recker 1.0.79 → 1.0.80-next.70ea84d

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/dist/browser/browser/mini.d.ts +2 -2
  2. package/dist/browser/browser/recker-mini.d.ts +8 -8
  3. package/dist/browser/browser/recker.d.ts +11 -8
  4. package/dist/browser/browser/recker.js +54 -6
  5. package/dist/browser/core/client.d.ts +15 -10
  6. package/dist/browser/core/client.js +54 -38
  7. package/dist/browser/index.iife.min.js +129 -130
  8. package/dist/browser/index.min.js +129 -130
  9. package/dist/browser/index.mini.iife.js +5697 -636
  10. package/dist/browser/index.mini.iife.min.js +47 -48
  11. package/dist/browser/index.mini.min.js +47 -48
  12. package/dist/browser/index.mini.umd.js +5697 -636
  13. package/dist/browser/index.mini.umd.min.js +47 -48
  14. package/dist/browser/index.umd.min.js +129 -130
  15. package/dist/browser/mini.d.ts +2 -2
  16. package/dist/browser/plugins/proxy-rotator.d.ts +2 -2
  17. package/dist/browser/plugins/proxy-rotator.js +6 -28
  18. package/dist/browser/recker-mini.d.ts +8 -8
  19. package/dist/browser/recker.d.ts +11 -8
  20. package/dist/browser/recker.js +54 -6
  21. package/dist/browser/scrape/document.js +2 -2
  22. package/dist/browser/scrape/element.js +7 -1
  23. package/dist/browser/scrape/parser/nodes/html.js +1 -1
  24. package/dist/browser/scrape/spider.d.ts +52 -0
  25. package/dist/browser/scrape/spider.js +620 -38
  26. package/dist/browser/scrape/types.d.ts +2 -0
  27. package/dist/browser/search/google.d.ts +26 -1
  28. package/dist/browser/search/google.js +427 -45
  29. package/dist/browser/seo/analyzer.d.ts +1 -0
  30. package/dist/browser/seo/analyzer.js +144 -1
  31. package/dist/browser/seo/index.d.ts +1 -1
  32. package/dist/browser/seo/keyword-campaign-analyzer.d.ts +2 -0
  33. package/dist/browser/seo/keyword-campaign-analyzer.js +538 -0
  34. package/dist/browser/seo/keyword-campaign-seed-advanced.d.ts +17 -0
  35. package/dist/browser/seo/keyword-campaign-seed-advanced.js +269 -0
  36. package/dist/browser/seo/keyword-campaign-seed-core.d.ts +29 -0
  37. package/dist/browser/seo/keyword-campaign-seed-core.js +525 -0
  38. package/dist/browser/seo/keyword-campaign-shared.d.ts +165 -0
  39. package/dist/browser/seo/keyword-campaign-shared.js +59 -0
  40. package/dist/browser/seo/keyword-campaign.d.ts +4 -107
  41. package/dist/browser/seo/keyword-campaign.js +2 -380
  42. package/dist/browser/seo/keywords.js +5 -22
  43. package/dist/browser/seo/types.d.ts +19 -0
  44. package/dist/browser/transport/curl.d.ts +5 -1
  45. package/dist/browser/transport/curl.js +207 -50
  46. package/dist/browser/transport/undici.d.ts +4 -3
  47. package/dist/browser/transport/undici.js +123 -49
  48. package/dist/browser/types/index.d.ts +9 -3
  49. package/dist/browser/utils/binary-manager.js +26 -3
  50. package/dist/browser/utils/block-detector.d.ts +8 -0
  51. package/dist/browser/utils/block-detector.js +542 -7
  52. package/dist/cli/commands/hls-runner.js +5 -4
  53. package/dist/cli/commands/live-runner.js +5 -4
  54. package/dist/cli/commands/loadtest-runner.js +3 -2
  55. package/dist/cli/commands/search.d.ts +2 -0
  56. package/dist/cli/commands/search.js +105 -0
  57. package/dist/cli/commands/seo-runner.js +9 -7
  58. package/dist/cli/commands/seo.js +140 -1
  59. package/dist/cli/commands/serve.js +75 -131
  60. package/dist/cli/commands/server-runner.js +59 -82
  61. package/dist/cli/commands/spider-runner.d.ts +37 -1
  62. package/dist/cli/commands/spider-runner.js +134 -10
  63. package/dist/cli/commands/spider.d.ts +18 -1
  64. package/dist/cli/commands/spider.js +457 -27
  65. package/dist/cli/events/handlers/cli.js +30 -1
  66. package/dist/cli/events/handlers/tui.js +26 -0
  67. package/dist/cli/events/types.d.ts +27 -0
  68. package/dist/cli/handler.d.ts +2 -12
  69. package/dist/cli/handler.js +20 -15
  70. package/dist/cli/handlers/protocols.js +39 -12
  71. package/dist/cli/handlers/search.d.ts +2 -0
  72. package/dist/cli/handlers/search.js +171 -0
  73. package/dist/cli/handlers/seo-analyze.d.ts +1 -0
  74. package/dist/cli/handlers/seo-analyze.js +666 -0
  75. package/dist/cli/handlers/seo-robots.d.ts +1 -0
  76. package/dist/cli/handlers/seo-robots.js +76 -0
  77. package/dist/cli/handlers/seo-serp.d.ts +54 -0
  78. package/dist/cli/handlers/seo-serp.js +243 -0
  79. package/dist/cli/handlers/seo-sitemap.d.ts +1 -0
  80. package/dist/cli/handlers/seo-sitemap.js +55 -0
  81. package/dist/cli/handlers/seo-spider.d.ts +1 -0
  82. package/dist/cli/handlers/seo-spider.js +334 -0
  83. package/dist/cli/handlers/seo.d.ts +8 -4
  84. package/dist/cli/handlers/seo.js +294 -442
  85. package/dist/cli/handlers/spider.js +94 -17
  86. package/dist/cli/handlers/streaming.js +5 -1
  87. package/dist/cli/index.js +11 -2
  88. package/dist/cli/presets.d.ts +1 -1
  89. package/dist/cli/presets.js +15 -4
  90. package/dist/cli/tui/app.js +6 -1
  91. package/dist/cli/tui/components/rich-response.d.ts +72 -0
  92. package/dist/cli/tui/components/rich-response.js +117 -0
  93. package/dist/cli/tui/executor-commands/background.js +30 -24
  94. package/dist/cli/tui/executor-commands/testing.js +3 -2
  95. package/dist/cli/tui/hooks/useDomains.d.ts +17 -0
  96. package/dist/cli/tui/hooks/useHelp.js +15 -2
  97. package/dist/cli/tui/job-manager.d.ts +4 -4
  98. package/dist/cli/tui/job-manager.js +5 -1
  99. package/dist/cli/tui/spider-tui.d.ts +63 -0
  100. package/dist/cli/tui/spider-tui.js +120 -2
  101. package/dist/cli/types.d.ts +12 -0
  102. package/dist/cli/types.js +1 -0
  103. package/dist/cli/utils/option-helpers.d.ts +10 -0
  104. package/dist/cli/utils/option-helpers.js +63 -0
  105. package/dist/cli/utils/score-color.d.ts +11 -0
  106. package/dist/cli/utils/score-color.js +11 -0
  107. package/dist/cli/utils/serp-campaign.d.ts +53 -0
  108. package/dist/cli/utils/serp-campaign.js +53 -0
  109. package/dist/cli/utils/serp-config.d.ts +26 -0
  110. package/dist/cli/utils/serp-config.js +125 -0
  111. package/dist/core/client.d.ts +15 -10
  112. package/dist/core/client.js +54 -38
  113. package/dist/index.d.ts +1 -0
  114. package/dist/index.js +1 -0
  115. package/dist/mcp/cli.js +35 -34
  116. package/dist/mcp/client.d.ts +2 -2
  117. package/dist/mcp/client.js +20 -2
  118. package/dist/mcp/contract.d.ts +1 -1
  119. package/dist/mcp/profiles.js +5 -1
  120. package/dist/mcp/prompts/index.js +8 -4
  121. package/dist/mcp/resources/index.js +46 -23
  122. package/dist/mcp/server.js +9 -6
  123. package/dist/mcp/tools/protocols.js +9 -2
  124. package/dist/mcp/tools/registry.js +13 -3
  125. package/dist/mcp/tools/seo.js +427 -2
  126. package/dist/mcp/types.d.ts +5 -1
  127. package/dist/plugins/proxy-rotator.d.ts +2 -2
  128. package/dist/plugins/proxy-rotator.js +6 -28
  129. package/dist/raffel/client.d.ts +38 -0
  130. package/dist/raffel/client.js +282 -0
  131. package/dist/raffel/index.d.ts +2 -0
  132. package/dist/raffel/index.js +2 -0
  133. package/dist/raffel/types.d.ts +40 -0
  134. package/dist/raffel/types.js +14 -0
  135. package/dist/recker.d.ts +13 -7
  136. package/dist/recker.js +58 -6
  137. package/dist/scrape/document.js +2 -2
  138. package/dist/scrape/element.js +7 -1
  139. package/dist/scrape/parser/nodes/html.js +1 -1
  140. package/dist/scrape/spider.d.ts +52 -0
  141. package/dist/scrape/spider.js +620 -38
  142. package/dist/scrape/types.d.ts +2 -0
  143. package/dist/search/google.d.ts +26 -1
  144. package/dist/search/google.js +427 -45
  145. package/dist/search/index.d.ts +1 -1
  146. package/dist/seo/analyzer.d.ts +1 -0
  147. package/dist/seo/analyzer.js +144 -1
  148. package/dist/seo/index.d.ts +1 -1
  149. package/dist/seo/keyword-campaign-analyzer.d.ts +2 -0
  150. package/dist/seo/keyword-campaign-analyzer.js +538 -0
  151. package/dist/seo/keyword-campaign-seed-advanced.d.ts +17 -0
  152. package/dist/seo/keyword-campaign-seed-advanced.js +269 -0
  153. package/dist/seo/keyword-campaign-seed-core.d.ts +29 -0
  154. package/dist/seo/keyword-campaign-seed-core.js +525 -0
  155. package/dist/seo/keyword-campaign-shared.d.ts +165 -0
  156. package/dist/seo/keyword-campaign-shared.js +59 -0
  157. package/dist/seo/keyword-campaign.d.ts +4 -107
  158. package/dist/seo/keyword-campaign.js +2 -380
  159. package/dist/seo/keywords.js +5 -22
  160. package/dist/seo/types.d.ts +19 -0
  161. package/dist/template/index.d.ts +1 -1
  162. package/dist/template/types.d.ts +0 -2
  163. package/dist/testing/index.d.ts +1 -22
  164. package/dist/testing/index.js +1 -11
  165. package/dist/transport/curl.d.ts +5 -1
  166. package/dist/transport/curl.js +207 -50
  167. package/dist/transport/undici.d.ts +4 -3
  168. package/dist/transport/undici.js +123 -49
  169. package/dist/types/index.d.ts +9 -3
  170. package/dist/utils/binary-manager.js +26 -3
  171. package/dist/utils/block-detector.d.ts +8 -0
  172. package/dist/utils/block-detector.js +542 -7
  173. package/dist/version.js +1 -1
  174. package/package.json +12 -1
  175. package/dist/testing/mock-dns-server.d.ts +0 -69
  176. package/dist/testing/mock-dns-server.js +0 -269
  177. package/dist/testing/mock-ftp-server.d.ts +0 -89
  178. package/dist/testing/mock-ftp-server.js +0 -562
  179. package/dist/testing/mock-hls-server.d.ts +0 -80
  180. package/dist/testing/mock-hls-server.js +0 -381
  181. package/dist/testing/mock-http-server.d.ts +0 -124
  182. package/dist/testing/mock-http-server.js +0 -343
  183. package/dist/testing/mock-proxy-server.d.ts +0 -108
  184. package/dist/testing/mock-proxy-server.js +0 -615
  185. package/dist/testing/mock-sse-server.d.ts +0 -76
  186. package/dist/testing/mock-sse-server.js +0 -291
  187. package/dist/testing/mock-telnet-server.d.ts +0 -59
  188. package/dist/testing/mock-telnet-server.js +0 -274
  189. package/dist/testing/mock-udp-server.d.ts +0 -43
  190. package/dist/testing/mock-udp-server.js +0 -188
  191. package/dist/testing/mock-websocket-server.d.ts +0 -76
  192. package/dist/testing/mock-websocket-server.js +0 -334
  193. package/dist/testing/mock-whois-server.d.ts +0 -56
  194. package/dist/testing/mock-whois-server.js +0 -234
  195. package/dist/testing/proxy-certs.d.ts +0 -19
  196. package/dist/testing/proxy-certs.js +0 -208
@@ -448,10 +448,10 @@ function startProxyJob(args, fullCommand) {
448
448
  }
449
449
  async function runProxyAsync(jobId, options, signal) {
450
450
  try {
451
- const { MockProxyServer } = await import('../../../testing/mock-proxy-server.js');
451
+ const { createMockProxyServer } = await import('../../../testing/index.js');
452
452
  const { setProxyRunning, setProxyStopped, addProxyRequest, updateProxyStats, } = await import('../hooks/useProxy.js');
453
453
  const manager = getJobManager();
454
- const proxy = await MockProxyServer.create({
454
+ const proxy = await createMockProxyServer({
455
455
  port: options.port,
456
456
  host: options.host,
457
457
  mode: options.intercept ? 'intercept' : 'forward',
@@ -468,46 +468,52 @@ async function runProxyAsync(jobId, options, signal) {
468
468
  let bytesIn = 0;
469
469
  let bytesOut = 0;
470
470
  let totalLatency = 0;
471
- proxy.on('request', (req) => {
471
+ proxy.on('request', ({ request, response }) => {
472
472
  requestCount++;
473
+ successCount++;
473
474
  const requestId = `${Date.now()}-${requestCount}`;
475
+ const url = request.url ?? '';
476
+ const isHttps = url.startsWith('https');
477
+ if (response?.body)
478
+ bytesIn += response.body.length || 0;
474
479
  addProxyRequest({
475
480
  id: requestId,
476
481
  timestamp: Date.now(),
477
- method: req.method,
478
- url: req.url,
479
- targetHost: req.targetHost,
480
- targetPort: req.targetPort,
481
- isHttps: req.isHttps,
482
- clientIp: req.clientIp,
483
- headers: req.headers,
482
+ method: request.method ?? 'GET',
483
+ url,
484
+ targetHost: (() => { try {
485
+ return new URL(url).hostname;
486
+ }
487
+ catch {
488
+ return url;
489
+ } })(),
490
+ targetPort: (() => { try {
491
+ return parseInt(new URL(url).port || (isHttps ? '443' : '80'));
492
+ }
493
+ catch {
494
+ return 80;
495
+ } })(),
496
+ isHttps,
497
+ clientIp: '',
498
+ headers: request.headers ?? {},
484
499
  });
485
500
  if (options.verbose) {
486
501
  manager.addLog(jobId, {
487
502
  timestamp: Date.now(),
488
503
  level: 'info',
489
- message: `${req.method} ${req.url}`,
504
+ message: `${request.method ?? 'GET'} ${url}`,
490
505
  });
491
506
  }
492
- manager.updateProgress(jobId, {
493
- proxyRequests: requestCount,
494
- proxySuccess: successCount,
495
- proxyErrors: errorCount,
496
- });
497
- });
498
- proxy.on('response', (res) => {
499
- successCount++;
500
- if (res.size)
501
- bytesIn += res.size;
507
+ const stats = proxy.statistics;
502
508
  updateProxyStats({
503
- totalRequests: requestCount,
509
+ totalRequests: stats.totalRequests,
504
510
  successCount,
505
511
  errorCount,
506
512
  bytesIn,
507
513
  bytesOut,
508
- avgLatency: successCount > 0 ? Math.round(totalLatency / successCount) : 0,
514
+ avgLatency: 0,
509
515
  requestsPerSecond: 0,
510
- activeConnections: proxy.stats.activeConnections,
516
+ activeConnections: 0,
511
517
  byStatusCode: {},
512
518
  byMethod: {},
513
519
  topHosts: [],
@@ -1,6 +1,7 @@
1
1
  import { parseArgValueFlag } from './parser.js';
2
2
  import { addHistoryItem, setPendingTabSwitch, } from '../hooks/useShellState.js';
3
3
  import { registerLoadTest, startJob, failJob, updateJobStatus, getJobManager, } from '../hooks/useJobs.js';
4
+ import { AbortError } from '../../../core/errors.js';
4
5
  export async function cmdLoad(ctx, args) {
5
6
  let url = args[0];
6
7
  if (!url) {
@@ -87,7 +88,7 @@ async function runLoadTestAsync(jobId, url, options, signal) {
87
88
  updateJobStatus(jobId, 'completed');
88
89
  }
89
90
  catch (err) {
90
- if (signal.aborted || err.message === 'Aborted') {
91
+ if (signal.aborted || err instanceof AbortError) {
91
92
  manager.addLog(jobId, {
92
93
  timestamp: Date.now(),
93
94
  level: 'info',
@@ -96,7 +97,7 @@ async function runLoadTestAsync(jobId, url, options, signal) {
96
97
  updateJobStatus(jobId, 'stopped');
97
98
  }
98
99
  else {
99
- failJob(jobId, err instanceof Error ? err : new Error(err.message));
100
+ failJob(jobId, err instanceof Error ? err : new Error('Load test failed'));
100
101
  }
101
102
  }
102
103
  }
@@ -24,6 +24,22 @@ export interface SeoIntelligence {
24
24
  };
25
25
  lastAnalyzed: Date;
26
26
  }
27
+ export interface SpiderSecuritySnapshot {
28
+ pages: number;
29
+ blockedPages: number;
30
+ captchaPages: number;
31
+ captchaProviders?: Record<string, number>;
32
+ attempts: number;
33
+ retries: number;
34
+ transportUsage?: {
35
+ undici?: number;
36
+ curl?: number;
37
+ };
38
+ avgAttempts?: number;
39
+ avgTtfbMs?: number;
40
+ avgTotalMs?: number;
41
+ avgDownloadMs?: number;
42
+ }
27
43
  export interface SpiderIntelligence {
28
44
  pagesFound: number;
29
45
  externalLinks: number;
@@ -32,6 +48,7 @@ export interface SpiderIntelligence {
32
48
  scripts: number;
33
49
  stylesheets: number;
34
50
  errors: number;
51
+ security?: SpiderSecuritySnapshot;
35
52
  lastCrawled: Date;
36
53
  }
37
54
  export interface DownloadEntry {
@@ -1,4 +1,5 @@
1
1
  import { createSignal } from 'tuiuiu.js';
2
+ import { StateError } from '../../../core/errors.js';
2
3
  const [helpQuery, setHelpQuery] = createSignal('');
3
4
  const [helpResults, setHelpResults] = createSignal([]);
4
5
  const [helpLoading, setHelpLoading] = createSignal(false);
@@ -21,7 +22,10 @@ async function getShellSearch() {
21
22
  return shellSearch;
22
23
  }
23
24
  catch (error) {
24
- throw new Error(`Failed to load search: ${error}`);
25
+ throw new StateError(`Failed to load search module for help shell.`, {
26
+ expectedState: 'ShellSearch module available',
27
+ actualState: getErrorMessage(error),
28
+ });
25
29
  }
26
30
  }
27
31
  async function searchHelp(query) {
@@ -70,7 +74,7 @@ async function searchHelp(query) {
70
74
  }
71
75
  }
72
76
  catch (error) {
73
- setHelpError(error.message || 'Search failed');
77
+ setHelpError(getErrorMessage(error));
74
78
  setHelpResults([]);
75
79
  }
76
80
  finally {
@@ -142,6 +146,15 @@ async function loadResultPreview(result) {
142
146
  setHelpPreviewContent(result.snippet);
143
147
  }
144
148
  }
149
+ function getErrorMessage(error) {
150
+ if (error instanceof Error) {
151
+ return error.message;
152
+ }
153
+ if (typeof error === 'string') {
154
+ return error;
155
+ }
156
+ return 'Search failed';
157
+ }
145
158
  function selectNextHelpResult() {
146
159
  const results = helpResults();
147
160
  if (results.length === 0)
@@ -5,7 +5,7 @@ export interface JobLogEntry {
5
5
  timestamp: number;
6
6
  level: 'info' | 'warn' | 'error' | 'debug';
7
7
  message: string;
8
- data?: any;
8
+ data?: unknown;
9
9
  }
10
10
  export interface JobProgress {
11
11
  segments: number;
@@ -41,7 +41,7 @@ export interface Job {
41
41
  command?: string;
42
42
  logs: JobLogEntry[];
43
43
  port?: number;
44
- metadata?: Record<string, any>;
44
+ metadata?: Record<string, unknown>;
45
45
  }
46
46
  export interface DownloadJobOptions {
47
47
  outputDir?: string;
@@ -70,7 +70,7 @@ export interface ServerJobOptions {
70
70
  type: 'http' | 'ws' | 'dns' | 'hls' | 'sse' | 'ftp' | 'telnet' | 'whois' | 'udp';
71
71
  port: number;
72
72
  host?: string;
73
- options?: Record<string, any>;
73
+ options?: Record<string, unknown>;
74
74
  }
75
75
  export interface LoadTestJobOptions {
76
76
  url: string;
@@ -149,7 +149,7 @@ export declare class JobManager extends EventEmitter {
149
149
  setOutput(id: number, output: string): void;
150
150
  setCommand(id: number, command: string): void;
151
151
  setPort(id: number, port: number): void;
152
- setMetadata(id: number, metadata: Record<string, any>): void;
152
+ setMetadata(id: number, metadata: Record<string, unknown>): void;
153
153
  start(id: number): void;
154
154
  fail(id: number, error: Error): void;
155
155
  emitLive(id: number): void;
@@ -1,4 +1,5 @@
1
1
  import { EventEmitter } from 'node:events';
2
+ import { StateError } from '../../core/errors.js';
2
3
  const MAX_LOGS_PER_JOB = 1000;
3
4
  export class JobManager extends EventEmitter {
4
5
  jobs = new Map();
@@ -40,7 +41,10 @@ export class JobManager extends EventEmitter {
40
41
  createJob(type, url, abortFn) {
41
42
  const activeCount = this.getActiveJobCount();
42
43
  if (activeCount >= this.maxJobs) {
43
- throw new Error(`Maximum concurrent active jobs (${this.maxJobs}) reached`);
44
+ throw new StateError(`Maximum concurrent active jobs (${this.maxJobs}) reached`, {
45
+ expectedState: `active jobs < ${this.maxJobs}`,
46
+ actualState: `active jobs = ${activeCount}`,
47
+ });
44
48
  }
45
49
  const job = {
46
50
  id: this.nextId++,
@@ -6,6 +6,41 @@ export interface SpiderTimings {
6
6
  download?: number;
7
7
  total?: number;
8
8
  }
9
+ export interface SpiderTransportUsage {
10
+ undici: number;
11
+ curl: number;
12
+ }
13
+ export interface SpiderSecuritySnapshot {
14
+ pages: number;
15
+ blockedPages: number;
16
+ captchaPages: number;
17
+ captchaProviders?: Record<string, number>;
18
+ attempts: number;
19
+ retries: number;
20
+ transportUsage: SpiderTransportUsage;
21
+ avgAttempts: number;
22
+ avgTtfbMs?: number;
23
+ avgTotalMs?: number;
24
+ avgDownloadMs?: number;
25
+ }
26
+ export interface SpiderSerpSummary {
27
+ queriesRequested: number;
28
+ queriesFound: number;
29
+ avgTopPosition?: number | null;
30
+ top3Count: number;
31
+ top10Count: number;
32
+ topOrganicCompetitors?: Array<{
33
+ domain: string;
34
+ matchedKeywords: number;
35
+ totalOutperformedQueries: number;
36
+ }>;
37
+ topPages?: Array<{
38
+ pageUrl: string;
39
+ found: number;
40
+ tracked: number;
41
+ appearanceRate: number;
42
+ }>;
43
+ }
9
44
  export interface PageInfo {
10
45
  url: string;
11
46
  score: number;
@@ -36,11 +71,39 @@ export interface SpiderTuiState {
36
71
  duplicateDescriptions: number;
37
72
  orphanPages: number;
38
73
  pagesWithErrors: number;
74
+ security: SpiderSecuritySnapshot;
75
+ serpSummary?: SpiderSerpSummary;
76
+ antiBotPages: number;
77
+ antiBotBlockedPages: number;
78
+ antiBotCaptchaPages: number;
79
+ antiBotCaptchaProviders: Record<string, number>;
80
+ antiBotAttempts: number;
81
+ antiBotRetries: number;
82
+ antiBotTransport: SpiderTransportUsage;
83
+ antiBotTtfbSamples: number;
84
+ antiBotTotalSamples: number;
85
+ antiBotDownloadSamples: number;
86
+ antiBotTtfbSum: number;
87
+ antiBotTotalSum: number;
88
+ antiBotDownloadSum: number;
39
89
  }
40
90
  export declare function createSpiderTui(startUrl: string, limit: number, concurrency: number): {
41
91
  updateProgress: (crawled: number, queued: number, pending: number, depth: number) => void;
42
92
  updateSeo: (url: string, score: number, bytes?: number, timings?: SpiderTimings) => void;
43
93
  updateUrl: (url: string) => void;
94
+ updateSecurity: (security?: {
95
+ blocked?: boolean;
96
+ captchaDetected?: boolean;
97
+ captchaProvider?: string;
98
+ attempts?: number;
99
+ retryCount?: number;
100
+ transport?: "undici" | "curl";
101
+ ttfb?: number;
102
+ total?: number;
103
+ download?: number;
104
+ }) => void;
105
+ setSecuritySummary: (summary: SpiderSecuritySnapshot) => void;
106
+ setSerpSummary: (serpSummary: SpiderSerpSummary) => void;
44
107
  setComplete: (summary: Partial<SpiderTuiState>) => void;
45
108
  setError: (error: string) => void;
46
109
  stop: () => void;
@@ -23,6 +23,34 @@ const initialState = {
23
23
  duplicateDescriptions: 0,
24
24
  orphanPages: 0,
25
25
  pagesWithErrors: 0,
26
+ security: {
27
+ pages: 0,
28
+ blockedPages: 0,
29
+ captchaPages: 0,
30
+ attempts: 0,
31
+ retries: 0,
32
+ transportUsage: {
33
+ undici: 0,
34
+ curl: 0,
35
+ },
36
+ avgAttempts: 0,
37
+ },
38
+ antiBotPages: 0,
39
+ antiBotBlockedPages: 0,
40
+ antiBotCaptchaPages: 0,
41
+ antiBotCaptchaProviders: {},
42
+ antiBotAttempts: 0,
43
+ antiBotRetries: 0,
44
+ antiBotTransport: {
45
+ undici: 0,
46
+ curl: 0,
47
+ },
48
+ antiBotTtfbSamples: 0,
49
+ antiBotTotalSamples: 0,
50
+ antiBotDownloadSamples: 0,
51
+ antiBotTtfbSum: 0,
52
+ antiBotTotalSum: 0,
53
+ antiBotDownloadSum: 0,
26
54
  };
27
55
  const SPARKLINE_SIZE = 30;
28
56
  const RECENT_PAGES_SIZE = 15;
@@ -79,6 +107,23 @@ function renderTimings(s) {
79
107
  }
80
108
  return timingRows;
81
109
  }
110
+ function formatSecurityRate(value, total) {
111
+ if (total <= 0)
112
+ return '0%';
113
+ return `${Math.round((value / total) * 100)}%`;
114
+ }
115
+ function safeNumber(value) {
116
+ return typeof value === 'number' && Number.isFinite(value) ? Math.round(value) : undefined;
117
+ }
118
+ function normalizeAttemptCount(value) {
119
+ if (typeof value === 'number' && Number.isFinite(value)) {
120
+ return Math.max(1, Math.floor(value));
121
+ }
122
+ return 1;
123
+ }
124
+ function hasTimingSample(value) {
125
+ return typeof value === 'number' && Number.isFinite(value);
126
+ }
82
127
  function renderRecentPages(pages, isComplete, totalAnalyzed) {
83
128
  if (pages.length === 0 || isComplete)
84
129
  return [];
@@ -174,8 +219,24 @@ export function createSpiderTui(startUrl, limit, concurrency) {
174
219
  const isComplete = s.status === 'complete';
175
220
  const totalToAnalyze = s.analyzed + s.queued + s.pending;
176
221
  const analysisPercent = totalToAnalyze > 0 ? Math.min(100, Math.round((s.analyzed / totalToAnalyze) * 100)) : 0;
222
+ const antiBotTotal = s.antiBotPages || s.analyzed || 1;
223
+ const blockedRate = formatSecurityRate(s.antiBotBlockedPages, antiBotTotal);
224
+ const captchaRate = formatSecurityRate(s.antiBotCaptchaPages, antiBotTotal);
225
+ const transportTotal = s.antiBotTransport.undici + s.antiBotTransport.curl;
226
+ const avgAttempts = antiBotTotal > 0 ? (s.antiBotAttempts / antiBotTotal).toFixed(2) : '0.00';
227
+ const avgTtfb = s.antiBotTtfbSamples > 0 ? Math.round(s.antiBotTtfbSum / s.antiBotTtfbSamples) : 'n/a';
228
+ const avgTotal = s.antiBotTotalSamples > 0 ? Math.round(s.antiBotTotalSum / s.antiBotTotalSamples) : 'n/a';
229
+ const avgDownload = s.antiBotDownloadSamples > 0 ? Math.round(s.antiBotDownloadSum / s.antiBotDownloadSamples) : 'n/a';
230
+ const formatTiming = (value) => value === 'n/a' ? 'n/a' : `${value}ms`;
231
+ const curlUsage = transportTotal > 0 ? Math.round((s.antiBotTransport.curl / transportTotal) * 100) : 0;
232
+ const transportLine = transportTotal > 0
233
+ ? `curl ${curlUsage}% / undici ${100 - curlUsage}%`
234
+ : 'n/a';
235
+ const captchaProviderSummary = Object.entries(s.antiBotCaptchaProviders)
236
+ .map(([provider, count]) => `${provider}: ${count}`)
237
+ .join(' | ');
177
238
  const scoreColor = s.avgScore >= 80 ? 'success' : s.avgScore >= 60 ? 'warning' : 'destructive';
178
- return Box({ flexDirection: 'column', padding: 1 }, Box({ flexDirection: 'row', marginBottom: 1, height: 1 }, Box({ width: 2 }, StatusIndicator({ status: isComplete ? 'success' : 'info' })), Text({ bold: true }, `SEO Spider: ${startUrl}`)), Box({ flexDirection: 'row', gap: 3, marginBottom: 1 }, Box({ flexDirection: 'column' }, Text({ dim: true }, 'Avg Score'), Text({ color: scoreColor, bold: true }, s.avgScore > 0 ? String(s.avgScore) : '--')), Box({ flexDirection: 'column' }, Text({ dim: true }, 'To Crawl'), Text({ color: (s.queued + s.pending) > 0 ? 'yellow' : 'gray', bold: true }, String(s.queued + s.pending))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Parallel'), Text({ color: 'magenta', bold: true }, String(concurrency))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Depth'), Text({ bold: true }, String(s.depth))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Time'), Text({ bold: true }, `${elapsed}s`)), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Traffic'), Text({ color: 'cyan', bold: true }, formatBytes(s.totalBytes)))), Box({ flexDirection: 'row' }, Box({ width: 8 }, Text({ dim: true }, 'Progress')), ProgressBar({
239
+ return Box({ flexDirection: 'column', padding: 1 }, Box({ flexDirection: 'row', marginBottom: 1, height: 1 }, Box({ width: 2 }, StatusIndicator({ status: isComplete ? 'success' : 'info' })), Text({ bold: true }, `SEO Spider: ${startUrl}`)), Box({ flexDirection: 'row', gap: 1, marginBottom: 1 }, Text({ dim: true }, 'Transport:'), Text({ color: 'magenta', bold: true }, transportLine)), Box({ flexDirection: 'row', gap: 3, marginBottom: 1 }, Box({ flexDirection: 'column' }, Text({ dim: true }, 'Avg Score'), Text({ color: scoreColor, bold: true }, s.avgScore > 0 ? String(s.avgScore) : '--')), Box({ flexDirection: 'column' }, Text({ dim: true }, 'To Crawl'), Text({ color: (s.queued + s.pending) > 0 ? 'yellow' : 'gray', bold: true }, String(s.queued + s.pending))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Parallel'), Text({ color: 'magenta', bold: true }, String(concurrency))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Depth'), Text({ bold: true }, String(s.depth))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Time'), Text({ bold: true }, `${elapsed}s`)), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Traffic'), Text({ color: 'cyan', bold: true }, formatBytes(s.totalBytes)))), Box({ flexDirection: 'row', gap: 3, marginBottom: 1 }, Box({ flexDirection: 'column' }, Text({ dim: true }, 'Anti-bot'), Text({ color: s.antiBotBlockedPages > 0 ? 'warning' : 'success', bold: true }, `${s.antiBotBlockedPages}/${antiBotTotal} blocked (${blockedRate})`)), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Captcha'), Text({ color: s.antiBotCaptchaPages > 0 ? 'warning' : 'success', bold: true }, `${s.antiBotCaptchaPages} (${captchaRate})`)), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Captcha Providers'), Text({ color: 'blue', bold: true }, captchaProviderSummary || 'n/a')), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Attempts'), Text({ color: 'blue', bold: true }, `avg ${avgAttempts}`))), Box({ flexDirection: 'row', gap: 3, marginBottom: 1 }, Box({ flexDirection: 'column' }, Text({ dim: true }, 'Avg timing'), Text({ color: 'cyan', bold: true }, `ttfb ${formatTiming(avgTtfb)} · total ${formatTiming(avgTotal)}`), Text({ color: 'cyan', bold: true }, `download ${formatTiming(avgDownload)}`))), Box({ flexDirection: 'row' }, Box({ width: 8 }, Text({ dim: true }, 'Progress')), ProgressBar({
179
240
  value: analysisPercent,
180
241
  width: 25,
181
242
  showPercentage: true,
@@ -195,7 +256,15 @@ export function createSpiderTui(startUrl, limit, concurrency) {
195
256
  width: 25,
196
257
  color: 'cyan',
197
258
  }), Box({ marginLeft: 1 }, Text({ color: 'cyan' }, `${last(s.pageSizes) ?? 0}KB`))),
198
- ] : []), ...renderTimings(s), ...renderRecentPages(s.recentPages, isComplete, s.analyzed), ...(isComplete ? [
259
+ ] : []), ...renderTimings(s), ...renderRecentPages(s.recentPages, isComplete, s.analyzed), ...(s.serpSummary ? [
260
+ Box({ flexDirection: 'column', marginTop: 1 }, Divider({}), Box({ flexDirection: 'row', marginTop: 1, height: 1 }, Text({ bold: true }, 'SERP campaign'), Box({ width: 2 }, Text({}, '')), Text({ color: 'green' }, `queries ${s.serpSummary.queriesFound}/${s.serpSummary.queriesRequested}`), Box({ width: 2 }, Text({}, '')), Text({ color: 'blue' }, `top3 ${s.serpSummary.top3Count}`), Box({ width: 2 }, Text({}, '')), Text({ color: 'cyan' }, `top10 ${s.serpSummary.top10Count}`), s.serpSummary.avgTopPosition !== undefined && s.serpSummary.avgTopPosition !== null
261
+ ? Box({ width: 2 }, Text({ color: 'yellow' }, `avg pos ${Math.round(s.serpSummary.avgTopPosition)}`))
262
+ : null), s.serpSummary.topOrganicCompetitors && s.serpSummary.topOrganicCompetitors.length > 0
263
+ ? Box({ marginTop: 1 }, Text({ dim: true }, 'Top competitors'))
264
+ : null, ...(s.serpSummary.topOrganicCompetitors || []).slice(0, 3).map((item) => Box({ flexDirection: 'row' }, Text({ dim: true }, `${item.domain}`), Text({ dim: true }, ` wins ${item.totalOutperformedQueries}`), Text({ dim: true }, ` matched ${item.matchedKeywords}`))), s.serpSummary.topPages && s.serpSummary.topPages.length > 0
265
+ ? Box({ marginTop: 1 }, Text({ dim: true }, 'Pages outranking baseline'))
266
+ : null, ...(s.serpSummary.topPages || []).slice(0, 3).map((item) => Box({ flexDirection: 'row' }, Text({ dim: true }, `${item.pageUrl}`), Text({ dim: true }, `${item.found}/${item.tracked}`), Text({ dim: true }, `${Math.round(item.appearanceRate)}%`)))),
267
+ ] : []), ...(isComplete ? [
199
268
  Box({ flexDirection: 'column', marginTop: 1 }, Divider({}), Box({ flexDirection: 'row', gap: 2, marginTop: 1, height: 1 }, ...[
200
269
  s.duplicateTitles > 0 ? Badge({ label: `${s.duplicateTitles} dup titles`, color: 'destructive' }) : null,
201
270
  s.duplicateDescriptions > 0 ? Badge({ label: `${s.duplicateDescriptions} dup desc`, color: 'warning' }) : null,
@@ -209,6 +278,55 @@ export function createSpiderTui(startUrl, limit, concurrency) {
209
278
  updateProgress,
210
279
  updateSeo,
211
280
  updateUrl,
281
+ updateSecurity: (security) => {
282
+ setState((s) => ({
283
+ ...s,
284
+ antiBotPages: s.antiBotPages + 1,
285
+ antiBotBlockedPages: s.antiBotBlockedPages + (security?.blocked ? 1 : 0),
286
+ antiBotCaptchaPages: s.antiBotCaptchaPages + (security?.captchaDetected ? 1 : 0),
287
+ antiBotCaptchaProviders: security?.captchaDetected ? {
288
+ ...s.antiBotCaptchaProviders,
289
+ [(security.captchaProvider || 'unknown').trim().toLowerCase()]: (s.antiBotCaptchaProviders[(security.captchaProvider || 'unknown').trim().toLowerCase()] || 0) + 1,
290
+ } : s.antiBotCaptchaProviders,
291
+ antiBotAttempts: s.antiBotAttempts + normalizeAttemptCount(security?.attempts),
292
+ antiBotRetries: s.antiBotRetries + (security?.retryCount ?? 0),
293
+ antiBotTransport: {
294
+ undici: s.antiBotTransport.undici + (security?.transport === 'undici' ? 1 : 0),
295
+ curl: s.antiBotTransport.curl + (security?.transport === 'curl' ? 1 : 0),
296
+ },
297
+ antiBotTtfbSamples: s.antiBotTtfbSamples + (hasTimingSample(security?.ttfb) ? 1 : 0),
298
+ antiBotTotalSamples: s.antiBotTotalSamples + (hasTimingSample(security?.total) ? 1 : 0),
299
+ antiBotDownloadSamples: s.antiBotDownloadSamples + (hasTimingSample(security?.download) ? 1 : 0),
300
+ antiBotTtfbSum: s.antiBotTtfbSum + (safeNumber(security?.ttfb) ?? 0),
301
+ antiBotTotalSum: s.antiBotTotalSum + (safeNumber(security?.total) ?? 0),
302
+ antiBotDownloadSum: s.antiBotDownloadSum + (safeNumber(security?.download) ?? 0),
303
+ }));
304
+ },
305
+ setSecuritySummary: (summary) => {
306
+ setState((s) => ({
307
+ ...s,
308
+ security: summary,
309
+ antiBotPages: summary.pages,
310
+ antiBotBlockedPages: summary.blockedPages,
311
+ antiBotCaptchaPages: summary.captchaPages,
312
+ antiBotAttempts: summary.attempts,
313
+ antiBotRetries: summary.retries,
314
+ antiBotTransport: {
315
+ undici: summary.transportUsage.undici,
316
+ curl: summary.transportUsage.curl,
317
+ },
318
+ antiBotCaptchaProviders: summary.captchaProviders || {},
319
+ antiBotTtfbSamples: summary.avgTtfbMs === undefined ? 0 : summary.pages,
320
+ antiBotTtfbSum: summary.avgTtfbMs === undefined ? 0 : Math.round(summary.avgTtfbMs * summary.pages),
321
+ antiBotTotalSamples: summary.avgTotalMs === undefined ? 0 : summary.pages,
322
+ antiBotTotalSum: summary.avgTotalMs === undefined ? 0 : Math.round(summary.avgTotalMs * summary.pages),
323
+ antiBotDownloadSamples: summary.avgDownloadMs === undefined ? 0 : summary.pages,
324
+ antiBotDownloadSum: summary.avgDownloadMs === undefined ? 0 : Math.round(summary.avgDownloadMs * summary.pages),
325
+ }));
326
+ },
327
+ setSerpSummary: (serpSummary) => {
328
+ setState((s) => ({ ...s, serpSummary }));
329
+ },
212
330
  setComplete,
213
331
  setError,
214
332
  stop: () => {
@@ -0,0 +1,12 @@
1
+ import type { ClientOptions, Method, RequestOptions } from '../types/index.js';
2
+ export type CliRequestHeaders = Record<string, string>;
3
+ export type CliRequestOptions = Omit<RequestOptions, 'method' | 'headers' | 'body'> & {
4
+ method: Method;
5
+ url: string;
6
+ headers?: CliRequestHeaders;
7
+ body?: RequestOptions['json'] | RequestOptions['body'];
8
+ verbose?: boolean;
9
+ quiet?: boolean;
10
+ output?: string;
11
+ clientOptions?: ClientOptions;
12
+ };
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,10 @@
1
+ import type { SpiderTransport } from '../../scrape/spider.js';
2
+ export type { SpiderTransport };
3
+ type OptionBag = Record<string, unknown>;
4
+ export type SpiderFocusMode = 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
5
+ export declare function getOptionValue(ctxOptions: OptionBag, ...keys: string[]): unknown;
6
+ export declare function toOptionString(value: unknown): string | undefined;
7
+ export declare function toBoolean(value: unknown, fallback?: boolean): boolean;
8
+ export declare function toNonNegativeInt(value: unknown, fallback?: number): number;
9
+ export declare function toSpiderTransport(value: unknown): SpiderTransport | undefined;
10
+ export declare function toSpiderFocusMode(value: unknown): SpiderFocusMode;
@@ -0,0 +1,63 @@
1
+ const SPIDER_FOCUS_MODES = ['all', 'links', 'duplicates', 'security', 'ai', 'resources'];
2
+ export function getOptionValue(ctxOptions, ...keys) {
3
+ for (const key of keys) {
4
+ if (Object.prototype.hasOwnProperty.call(ctxOptions, key)) {
5
+ return ctxOptions[key];
6
+ }
7
+ }
8
+ return undefined;
9
+ }
10
+ export function toOptionString(value) {
11
+ if (typeof value === 'string') {
12
+ const cleaned = value.trim();
13
+ return cleaned.length > 0 ? cleaned : undefined;
14
+ }
15
+ if (typeof value === 'number')
16
+ return Number.isFinite(value) ? String(value) : undefined;
17
+ return undefined;
18
+ }
19
+ export function toBoolean(value, fallback = false) {
20
+ if (typeof value === 'boolean') {
21
+ return value;
22
+ }
23
+ if (typeof value === 'number') {
24
+ return value !== 0;
25
+ }
26
+ if (typeof value === 'string') {
27
+ const normalized = value.trim().toLowerCase();
28
+ if (normalized === '' || normalized === 'default') {
29
+ return fallback;
30
+ }
31
+ if (['1', 'true', 'on', 'yes'].includes(normalized)) {
32
+ return true;
33
+ }
34
+ if (['0', 'false', 'off', 'no', 'none'].includes(normalized)) {
35
+ return false;
36
+ }
37
+ return fallback;
38
+ }
39
+ if (value && typeof value === 'object' && 'default' in value) {
40
+ return toBoolean(value.default, fallback);
41
+ }
42
+ return fallback;
43
+ }
44
+ export function toNonNegativeInt(value, fallback = 0) {
45
+ const parsed = typeof value === 'number'
46
+ ? value
47
+ : typeof value === 'string'
48
+ ? Number(value.trim())
49
+ : fallback;
50
+ return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallback;
51
+ }
52
+ export function toSpiderTransport(value) {
53
+ const raw = toOptionString(value)?.toLowerCase();
54
+ if (raw === 'auto' || raw === 'undici' || raw === 'curl')
55
+ return raw;
56
+ return undefined;
57
+ }
58
+ export function toSpiderFocusMode(value) {
59
+ const raw = toOptionString(value)?.toLowerCase();
60
+ return raw && SPIDER_FOCUS_MODES.includes(raw)
61
+ ? raw
62
+ : 'all';
63
+ }
@@ -0,0 +1,11 @@
1
+ export type ScoreColor = (value: string) => string;
2
+ export type ScorePalette = {
3
+ green: ScoreColor;
4
+ yellow: ScoreColor;
5
+ red: ScoreColor;
6
+ };
7
+ export type ScoreColorThresholds = {
8
+ high: number;
9
+ medium: number;
10
+ };
11
+ export declare function getScoreColor(score: number, colors: ScorePalette, thresholds?: ScoreColorThresholds): ScoreColor;
@@ -0,0 +1,11 @@
1
+ const DEFAULT_SCORE_THRESHOLDS = {
2
+ high: 80,
3
+ medium: 60,
4
+ };
5
+ export function getScoreColor(score, colors, thresholds = DEFAULT_SCORE_THRESHOLDS) {
6
+ if (score >= thresholds.high)
7
+ return colors.green;
8
+ if (score >= thresholds.medium)
9
+ return colors.yellow;
10
+ return colors.red;
11
+ }
@@ -0,0 +1,53 @@
1
+ import type { SpiderSerpOptionConfig } from './serp-config.js';
2
+ import type { SeoCrawlerSerpRun, SerpSeedPage } from '../handlers/seo.js';
3
+ export type SpiderSerpCampaignSeed = {
4
+ keyword: string;
5
+ sourcePage?: string;
6
+ sourceWeight: number;
7
+ };
8
+ export interface SpiderSerpCampaignPayload {
9
+ summary: {
10
+ queriesRequested: number;
11
+ queriesExecuted: number;
12
+ queriesFound: number;
13
+ avgTopPosition: number | null;
14
+ top3Count: number;
15
+ top10Count: number;
16
+ topOrganicCompetitors: Array<{
17
+ domain: string;
18
+ matchedKeywords: number;
19
+ totalOutperformedQueries?: number;
20
+ organicQueries?: number;
21
+ paidQueries?: number;
22
+ avgOutperformedGap?: number | null;
23
+ }>;
24
+ };
25
+ campaign: {
26
+ active: boolean;
27
+ confidence: 'high' | 'medium' | 'low';
28
+ evidence: string[];
29
+ };
30
+ results: Array<{
31
+ keyword: string;
32
+ found: boolean;
33
+ position: number | null;
34
+ targetUrl?: string;
35
+ searchUrl: string;
36
+ }>;
37
+ pageComparison: Array<{
38
+ pageUrl: string;
39
+ tracked: number;
40
+ found: number;
41
+ appearanceRate: string;
42
+ avgPosition: string;
43
+ top3: number;
44
+ top10: number;
45
+ }>;
46
+ seedPlan: {
47
+ short: SpiderSerpCampaignSeed[];
48
+ longTail: SpiderSerpCampaignSeed[];
49
+ ordered: SpiderSerpCampaignSeed[];
50
+ };
51
+ }
52
+ export declare function runSpiderKeywordCampaign(targetUrl: string, pages: SerpSeedPage[], config: SpiderSerpOptionConfig): Promise<SeoCrawlerSerpRun | undefined>;
53
+ export declare function toSerpPayload(campaign: SeoCrawlerSerpRun): SpiderSerpCampaignPayload;