recker 1.0.79 → 1.0.80-next.70ea84d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/browser/mini.d.ts +2 -2
- package/dist/browser/browser/recker-mini.d.ts +8 -8
- package/dist/browser/browser/recker.d.ts +11 -8
- package/dist/browser/browser/recker.js +54 -6
- package/dist/browser/core/client.d.ts +15 -10
- package/dist/browser/core/client.js +54 -38
- package/dist/browser/index.iife.min.js +129 -130
- package/dist/browser/index.min.js +129 -130
- package/dist/browser/index.mini.iife.js +5697 -636
- package/dist/browser/index.mini.iife.min.js +47 -48
- package/dist/browser/index.mini.min.js +47 -48
- package/dist/browser/index.mini.umd.js +5697 -636
- package/dist/browser/index.mini.umd.min.js +47 -48
- package/dist/browser/index.umd.min.js +129 -130
- package/dist/browser/mini.d.ts +2 -2
- package/dist/browser/plugins/proxy-rotator.d.ts +2 -2
- package/dist/browser/plugins/proxy-rotator.js +6 -28
- package/dist/browser/recker-mini.d.ts +8 -8
- package/dist/browser/recker.d.ts +11 -8
- package/dist/browser/recker.js +54 -6
- package/dist/browser/scrape/document.js +2 -2
- package/dist/browser/scrape/element.js +7 -1
- package/dist/browser/scrape/parser/nodes/html.js +1 -1
- package/dist/browser/scrape/spider.d.ts +52 -0
- package/dist/browser/scrape/spider.js +620 -38
- package/dist/browser/scrape/types.d.ts +2 -0
- package/dist/browser/search/google.d.ts +26 -1
- package/dist/browser/search/google.js +427 -45
- package/dist/browser/seo/analyzer.d.ts +1 -0
- package/dist/browser/seo/analyzer.js +144 -1
- package/dist/browser/seo/index.d.ts +1 -1
- package/dist/browser/seo/keyword-campaign-analyzer.d.ts +2 -0
- package/dist/browser/seo/keyword-campaign-analyzer.js +538 -0
- package/dist/browser/seo/keyword-campaign-seed-advanced.d.ts +17 -0
- package/dist/browser/seo/keyword-campaign-seed-advanced.js +269 -0
- package/dist/browser/seo/keyword-campaign-seed-core.d.ts +29 -0
- package/dist/browser/seo/keyword-campaign-seed-core.js +525 -0
- package/dist/browser/seo/keyword-campaign-shared.d.ts +165 -0
- package/dist/browser/seo/keyword-campaign-shared.js +59 -0
- package/dist/browser/seo/keyword-campaign.d.ts +4 -107
- package/dist/browser/seo/keyword-campaign.js +2 -380
- package/dist/browser/seo/keywords.js +5 -22
- package/dist/browser/seo/types.d.ts +19 -0
- package/dist/browser/transport/curl.d.ts +5 -1
- package/dist/browser/transport/curl.js +207 -50
- package/dist/browser/transport/undici.d.ts +4 -3
- package/dist/browser/transport/undici.js +123 -49
- package/dist/browser/types/index.d.ts +9 -3
- package/dist/browser/utils/binary-manager.js +26 -3
- package/dist/browser/utils/block-detector.d.ts +8 -0
- package/dist/browser/utils/block-detector.js +542 -7
- package/dist/cli/commands/hls-runner.js +5 -4
- package/dist/cli/commands/live-runner.js +5 -4
- package/dist/cli/commands/loadtest-runner.js +3 -2
- package/dist/cli/commands/search.d.ts +2 -0
- package/dist/cli/commands/search.js +105 -0
- package/dist/cli/commands/seo-runner.js +9 -7
- package/dist/cli/commands/seo.js +140 -1
- package/dist/cli/commands/serve.js +75 -131
- package/dist/cli/commands/server-runner.js +59 -82
- package/dist/cli/commands/spider-runner.d.ts +37 -1
- package/dist/cli/commands/spider-runner.js +134 -10
- package/dist/cli/commands/spider.d.ts +18 -1
- package/dist/cli/commands/spider.js +457 -27
- package/dist/cli/events/handlers/cli.js +30 -1
- package/dist/cli/events/handlers/tui.js +26 -0
- package/dist/cli/events/types.d.ts +27 -0
- package/dist/cli/handler.d.ts +2 -12
- package/dist/cli/handler.js +20 -15
- package/dist/cli/handlers/protocols.js +39 -12
- package/dist/cli/handlers/search.d.ts +2 -0
- package/dist/cli/handlers/search.js +171 -0
- package/dist/cli/handlers/seo-analyze.d.ts +1 -0
- package/dist/cli/handlers/seo-analyze.js +666 -0
- package/dist/cli/handlers/seo-robots.d.ts +1 -0
- package/dist/cli/handlers/seo-robots.js +76 -0
- package/dist/cli/handlers/seo-serp.d.ts +54 -0
- package/dist/cli/handlers/seo-serp.js +243 -0
- package/dist/cli/handlers/seo-sitemap.d.ts +1 -0
- package/dist/cli/handlers/seo-sitemap.js +55 -0
- package/dist/cli/handlers/seo-spider.d.ts +1 -0
- package/dist/cli/handlers/seo-spider.js +334 -0
- package/dist/cli/handlers/seo.d.ts +8 -4
- package/dist/cli/handlers/seo.js +294 -442
- package/dist/cli/handlers/spider.js +94 -17
- package/dist/cli/handlers/streaming.js +5 -1
- package/dist/cli/index.js +11 -2
- package/dist/cli/presets.d.ts +1 -1
- package/dist/cli/presets.js +15 -4
- package/dist/cli/tui/app.js +6 -1
- package/dist/cli/tui/components/rich-response.d.ts +72 -0
- package/dist/cli/tui/components/rich-response.js +117 -0
- package/dist/cli/tui/executor-commands/background.js +30 -24
- package/dist/cli/tui/executor-commands/testing.js +3 -2
- package/dist/cli/tui/hooks/useDomains.d.ts +17 -0
- package/dist/cli/tui/hooks/useHelp.js +15 -2
- package/dist/cli/tui/job-manager.d.ts +4 -4
- package/dist/cli/tui/job-manager.js +5 -1
- package/dist/cli/tui/spider-tui.d.ts +63 -0
- package/dist/cli/tui/spider-tui.js +120 -2
- package/dist/cli/types.d.ts +12 -0
- package/dist/cli/types.js +1 -0
- package/dist/cli/utils/option-helpers.d.ts +10 -0
- package/dist/cli/utils/option-helpers.js +63 -0
- package/dist/cli/utils/score-color.d.ts +11 -0
- package/dist/cli/utils/score-color.js +11 -0
- package/dist/cli/utils/serp-campaign.d.ts +53 -0
- package/dist/cli/utils/serp-campaign.js +53 -0
- package/dist/cli/utils/serp-config.d.ts +26 -0
- package/dist/cli/utils/serp-config.js +125 -0
- package/dist/core/client.d.ts +15 -10
- package/dist/core/client.js +54 -38
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp/cli.js +35 -34
- package/dist/mcp/client.d.ts +2 -2
- package/dist/mcp/client.js +20 -2
- package/dist/mcp/contract.d.ts +1 -1
- package/dist/mcp/profiles.js +5 -1
- package/dist/mcp/prompts/index.js +8 -4
- package/dist/mcp/resources/index.js +46 -23
- package/dist/mcp/server.js +9 -6
- package/dist/mcp/tools/protocols.js +9 -2
- package/dist/mcp/tools/registry.js +13 -3
- package/dist/mcp/tools/seo.js +427 -2
- package/dist/mcp/types.d.ts +5 -1
- package/dist/plugins/proxy-rotator.d.ts +2 -2
- package/dist/plugins/proxy-rotator.js +6 -28
- package/dist/raffel/client.d.ts +38 -0
- package/dist/raffel/client.js +282 -0
- package/dist/raffel/index.d.ts +2 -0
- package/dist/raffel/index.js +2 -0
- package/dist/raffel/types.d.ts +40 -0
- package/dist/raffel/types.js +14 -0
- package/dist/recker.d.ts +13 -7
- package/dist/recker.js +58 -6
- package/dist/scrape/document.js +2 -2
- package/dist/scrape/element.js +7 -1
- package/dist/scrape/parser/nodes/html.js +1 -1
- package/dist/scrape/spider.d.ts +52 -0
- package/dist/scrape/spider.js +620 -38
- package/dist/scrape/types.d.ts +2 -0
- package/dist/search/google.d.ts +26 -1
- package/dist/search/google.js +427 -45
- package/dist/search/index.d.ts +1 -1
- package/dist/seo/analyzer.d.ts +1 -0
- package/dist/seo/analyzer.js +144 -1
- package/dist/seo/index.d.ts +1 -1
- package/dist/seo/keyword-campaign-analyzer.d.ts +2 -0
- package/dist/seo/keyword-campaign-analyzer.js +538 -0
- package/dist/seo/keyword-campaign-seed-advanced.d.ts +17 -0
- package/dist/seo/keyword-campaign-seed-advanced.js +269 -0
- package/dist/seo/keyword-campaign-seed-core.d.ts +29 -0
- package/dist/seo/keyword-campaign-seed-core.js +525 -0
- package/dist/seo/keyword-campaign-shared.d.ts +165 -0
- package/dist/seo/keyword-campaign-shared.js +59 -0
- package/dist/seo/keyword-campaign.d.ts +4 -107
- package/dist/seo/keyword-campaign.js +2 -380
- package/dist/seo/keywords.js +5 -22
- package/dist/seo/types.d.ts +19 -0
- package/dist/template/index.d.ts +1 -1
- package/dist/template/types.d.ts +0 -2
- package/dist/testing/index.d.ts +1 -22
- package/dist/testing/index.js +1 -11
- package/dist/transport/curl.d.ts +5 -1
- package/dist/transport/curl.js +207 -50
- package/dist/transport/undici.d.ts +4 -3
- package/dist/transport/undici.js +123 -49
- package/dist/types/index.d.ts +9 -3
- package/dist/utils/binary-manager.js +26 -3
- package/dist/utils/block-detector.d.ts +8 -0
- package/dist/utils/block-detector.js +542 -7
- package/dist/version.js +1 -1
- package/package.json +12 -1
- package/dist/testing/mock-dns-server.d.ts +0 -69
- package/dist/testing/mock-dns-server.js +0 -269
- package/dist/testing/mock-ftp-server.d.ts +0 -89
- package/dist/testing/mock-ftp-server.js +0 -562
- package/dist/testing/mock-hls-server.d.ts +0 -80
- package/dist/testing/mock-hls-server.js +0 -381
- package/dist/testing/mock-http-server.d.ts +0 -124
- package/dist/testing/mock-http-server.js +0 -343
- package/dist/testing/mock-proxy-server.d.ts +0 -108
- package/dist/testing/mock-proxy-server.js +0 -615
- package/dist/testing/mock-sse-server.d.ts +0 -76
- package/dist/testing/mock-sse-server.js +0 -291
- package/dist/testing/mock-telnet-server.d.ts +0 -59
- package/dist/testing/mock-telnet-server.js +0 -274
- package/dist/testing/mock-udp-server.d.ts +0 -43
- package/dist/testing/mock-udp-server.js +0 -188
- package/dist/testing/mock-websocket-server.d.ts +0 -76
- package/dist/testing/mock-websocket-server.js +0 -334
- package/dist/testing/mock-whois-server.d.ts +0 -56
- package/dist/testing/mock-whois-server.js +0 -234
- package/dist/testing/proxy-certs.d.ts +0 -19
- package/dist/testing/proxy-certs.js +0 -208
|
@@ -448,10 +448,10 @@ function startProxyJob(args, fullCommand) {
|
|
|
448
448
|
}
|
|
449
449
|
async function runProxyAsync(jobId, options, signal) {
|
|
450
450
|
try {
|
|
451
|
-
const {
|
|
451
|
+
const { createMockProxyServer } = await import('../../../testing/index.js');
|
|
452
452
|
const { setProxyRunning, setProxyStopped, addProxyRequest, updateProxyStats, } = await import('../hooks/useProxy.js');
|
|
453
453
|
const manager = getJobManager();
|
|
454
|
-
const proxy = await
|
|
454
|
+
const proxy = await createMockProxyServer({
|
|
455
455
|
port: options.port,
|
|
456
456
|
host: options.host,
|
|
457
457
|
mode: options.intercept ? 'intercept' : 'forward',
|
|
@@ -468,46 +468,52 @@ async function runProxyAsync(jobId, options, signal) {
|
|
|
468
468
|
let bytesIn = 0;
|
|
469
469
|
let bytesOut = 0;
|
|
470
470
|
let totalLatency = 0;
|
|
471
|
-
proxy.on('request', (
|
|
471
|
+
proxy.on('request', ({ request, response }) => {
|
|
472
472
|
requestCount++;
|
|
473
|
+
successCount++;
|
|
473
474
|
const requestId = `${Date.now()}-${requestCount}`;
|
|
475
|
+
const url = request.url ?? '';
|
|
476
|
+
const isHttps = url.startsWith('https');
|
|
477
|
+
if (response?.body)
|
|
478
|
+
bytesIn += response.body.length || 0;
|
|
474
479
|
addProxyRequest({
|
|
475
480
|
id: requestId,
|
|
476
481
|
timestamp: Date.now(),
|
|
477
|
-
method:
|
|
478
|
-
url
|
|
479
|
-
targetHost:
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
482
|
+
method: request.method ?? 'GET',
|
|
483
|
+
url,
|
|
484
|
+
targetHost: (() => { try {
|
|
485
|
+
return new URL(url).hostname;
|
|
486
|
+
}
|
|
487
|
+
catch {
|
|
488
|
+
return url;
|
|
489
|
+
} })(),
|
|
490
|
+
targetPort: (() => { try {
|
|
491
|
+
return parseInt(new URL(url).port || (isHttps ? '443' : '80'));
|
|
492
|
+
}
|
|
493
|
+
catch {
|
|
494
|
+
return 80;
|
|
495
|
+
} })(),
|
|
496
|
+
isHttps,
|
|
497
|
+
clientIp: '',
|
|
498
|
+
headers: request.headers ?? {},
|
|
484
499
|
});
|
|
485
500
|
if (options.verbose) {
|
|
486
501
|
manager.addLog(jobId, {
|
|
487
502
|
timestamp: Date.now(),
|
|
488
503
|
level: 'info',
|
|
489
|
-
message: `${
|
|
504
|
+
message: `${request.method ?? 'GET'} ${url}`,
|
|
490
505
|
});
|
|
491
506
|
}
|
|
492
|
-
|
|
493
|
-
proxyRequests: requestCount,
|
|
494
|
-
proxySuccess: successCount,
|
|
495
|
-
proxyErrors: errorCount,
|
|
496
|
-
});
|
|
497
|
-
});
|
|
498
|
-
proxy.on('response', (res) => {
|
|
499
|
-
successCount++;
|
|
500
|
-
if (res.size)
|
|
501
|
-
bytesIn += res.size;
|
|
507
|
+
const stats = proxy.statistics;
|
|
502
508
|
updateProxyStats({
|
|
503
|
-
totalRequests:
|
|
509
|
+
totalRequests: stats.totalRequests,
|
|
504
510
|
successCount,
|
|
505
511
|
errorCount,
|
|
506
512
|
bytesIn,
|
|
507
513
|
bytesOut,
|
|
508
|
-
avgLatency:
|
|
514
|
+
avgLatency: 0,
|
|
509
515
|
requestsPerSecond: 0,
|
|
510
|
-
activeConnections:
|
|
516
|
+
activeConnections: 0,
|
|
511
517
|
byStatusCode: {},
|
|
512
518
|
byMethod: {},
|
|
513
519
|
topHosts: [],
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { parseArgValueFlag } from './parser.js';
|
|
2
2
|
import { addHistoryItem, setPendingTabSwitch, } from '../hooks/useShellState.js';
|
|
3
3
|
import { registerLoadTest, startJob, failJob, updateJobStatus, getJobManager, } from '../hooks/useJobs.js';
|
|
4
|
+
import { AbortError } from '../../../core/errors.js';
|
|
4
5
|
export async function cmdLoad(ctx, args) {
|
|
5
6
|
let url = args[0];
|
|
6
7
|
if (!url) {
|
|
@@ -87,7 +88,7 @@ async function runLoadTestAsync(jobId, url, options, signal) {
|
|
|
87
88
|
updateJobStatus(jobId, 'completed');
|
|
88
89
|
}
|
|
89
90
|
catch (err) {
|
|
90
|
-
if (signal.aborted || err
|
|
91
|
+
if (signal.aborted || err instanceof AbortError) {
|
|
91
92
|
manager.addLog(jobId, {
|
|
92
93
|
timestamp: Date.now(),
|
|
93
94
|
level: 'info',
|
|
@@ -96,7 +97,7 @@ async function runLoadTestAsync(jobId, url, options, signal) {
|
|
|
96
97
|
updateJobStatus(jobId, 'stopped');
|
|
97
98
|
}
|
|
98
99
|
else {
|
|
99
|
-
failJob(jobId, err instanceof Error ? err : new Error(
|
|
100
|
+
failJob(jobId, err instanceof Error ? err : new Error('Load test failed'));
|
|
100
101
|
}
|
|
101
102
|
}
|
|
102
103
|
}
|
|
@@ -24,6 +24,22 @@ export interface SeoIntelligence {
|
|
|
24
24
|
};
|
|
25
25
|
lastAnalyzed: Date;
|
|
26
26
|
}
|
|
27
|
+
export interface SpiderSecuritySnapshot {
|
|
28
|
+
pages: number;
|
|
29
|
+
blockedPages: number;
|
|
30
|
+
captchaPages: number;
|
|
31
|
+
captchaProviders?: Record<string, number>;
|
|
32
|
+
attempts: number;
|
|
33
|
+
retries: number;
|
|
34
|
+
transportUsage?: {
|
|
35
|
+
undici?: number;
|
|
36
|
+
curl?: number;
|
|
37
|
+
};
|
|
38
|
+
avgAttempts?: number;
|
|
39
|
+
avgTtfbMs?: number;
|
|
40
|
+
avgTotalMs?: number;
|
|
41
|
+
avgDownloadMs?: number;
|
|
42
|
+
}
|
|
27
43
|
export interface SpiderIntelligence {
|
|
28
44
|
pagesFound: number;
|
|
29
45
|
externalLinks: number;
|
|
@@ -32,6 +48,7 @@ export interface SpiderIntelligence {
|
|
|
32
48
|
scripts: number;
|
|
33
49
|
stylesheets: number;
|
|
34
50
|
errors: number;
|
|
51
|
+
security?: SpiderSecuritySnapshot;
|
|
35
52
|
lastCrawled: Date;
|
|
36
53
|
}
|
|
37
54
|
export interface DownloadEntry {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createSignal } from 'tuiuiu.js';
|
|
2
|
+
import { StateError } from '../../../core/errors.js';
|
|
2
3
|
const [helpQuery, setHelpQuery] = createSignal('');
|
|
3
4
|
const [helpResults, setHelpResults] = createSignal([]);
|
|
4
5
|
const [helpLoading, setHelpLoading] = createSignal(false);
|
|
@@ -21,7 +22,10 @@ async function getShellSearch() {
|
|
|
21
22
|
return shellSearch;
|
|
22
23
|
}
|
|
23
24
|
catch (error) {
|
|
24
|
-
throw new
|
|
25
|
+
throw new StateError(`Failed to load search module for help shell.`, {
|
|
26
|
+
expectedState: 'ShellSearch module available',
|
|
27
|
+
actualState: getErrorMessage(error),
|
|
28
|
+
});
|
|
25
29
|
}
|
|
26
30
|
}
|
|
27
31
|
async function searchHelp(query) {
|
|
@@ -70,7 +74,7 @@ async function searchHelp(query) {
|
|
|
70
74
|
}
|
|
71
75
|
}
|
|
72
76
|
catch (error) {
|
|
73
|
-
setHelpError(error
|
|
77
|
+
setHelpError(getErrorMessage(error));
|
|
74
78
|
setHelpResults([]);
|
|
75
79
|
}
|
|
76
80
|
finally {
|
|
@@ -142,6 +146,15 @@ async function loadResultPreview(result) {
|
|
|
142
146
|
setHelpPreviewContent(result.snippet);
|
|
143
147
|
}
|
|
144
148
|
}
|
|
149
|
+
function getErrorMessage(error) {
|
|
150
|
+
if (error instanceof Error) {
|
|
151
|
+
return error.message;
|
|
152
|
+
}
|
|
153
|
+
if (typeof error === 'string') {
|
|
154
|
+
return error;
|
|
155
|
+
}
|
|
156
|
+
return 'Search failed';
|
|
157
|
+
}
|
|
145
158
|
function selectNextHelpResult() {
|
|
146
159
|
const results = helpResults();
|
|
147
160
|
if (results.length === 0)
|
|
@@ -5,7 +5,7 @@ export interface JobLogEntry {
|
|
|
5
5
|
timestamp: number;
|
|
6
6
|
level: 'info' | 'warn' | 'error' | 'debug';
|
|
7
7
|
message: string;
|
|
8
|
-
data?:
|
|
8
|
+
data?: unknown;
|
|
9
9
|
}
|
|
10
10
|
export interface JobProgress {
|
|
11
11
|
segments: number;
|
|
@@ -41,7 +41,7 @@ export interface Job {
|
|
|
41
41
|
command?: string;
|
|
42
42
|
logs: JobLogEntry[];
|
|
43
43
|
port?: number;
|
|
44
|
-
metadata?: Record<string,
|
|
44
|
+
metadata?: Record<string, unknown>;
|
|
45
45
|
}
|
|
46
46
|
export interface DownloadJobOptions {
|
|
47
47
|
outputDir?: string;
|
|
@@ -70,7 +70,7 @@ export interface ServerJobOptions {
|
|
|
70
70
|
type: 'http' | 'ws' | 'dns' | 'hls' | 'sse' | 'ftp' | 'telnet' | 'whois' | 'udp';
|
|
71
71
|
port: number;
|
|
72
72
|
host?: string;
|
|
73
|
-
options?: Record<string,
|
|
73
|
+
options?: Record<string, unknown>;
|
|
74
74
|
}
|
|
75
75
|
export interface LoadTestJobOptions {
|
|
76
76
|
url: string;
|
|
@@ -149,7 +149,7 @@ export declare class JobManager extends EventEmitter {
|
|
|
149
149
|
setOutput(id: number, output: string): void;
|
|
150
150
|
setCommand(id: number, command: string): void;
|
|
151
151
|
setPort(id: number, port: number): void;
|
|
152
|
-
setMetadata(id: number, metadata: Record<string,
|
|
152
|
+
setMetadata(id: number, metadata: Record<string, unknown>): void;
|
|
153
153
|
start(id: number): void;
|
|
154
154
|
fail(id: number, error: Error): void;
|
|
155
155
|
emitLive(id: number): void;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
|
+
import { StateError } from '../../core/errors.js';
|
|
2
3
|
const MAX_LOGS_PER_JOB = 1000;
|
|
3
4
|
export class JobManager extends EventEmitter {
|
|
4
5
|
jobs = new Map();
|
|
@@ -40,7 +41,10 @@ export class JobManager extends EventEmitter {
|
|
|
40
41
|
createJob(type, url, abortFn) {
|
|
41
42
|
const activeCount = this.getActiveJobCount();
|
|
42
43
|
if (activeCount >= this.maxJobs) {
|
|
43
|
-
throw new
|
|
44
|
+
throw new StateError(`Maximum concurrent active jobs (${this.maxJobs}) reached`, {
|
|
45
|
+
expectedState: `active jobs < ${this.maxJobs}`,
|
|
46
|
+
actualState: `active jobs = ${activeCount}`,
|
|
47
|
+
});
|
|
44
48
|
}
|
|
45
49
|
const job = {
|
|
46
50
|
id: this.nextId++,
|
|
@@ -6,6 +6,41 @@ export interface SpiderTimings {
|
|
|
6
6
|
download?: number;
|
|
7
7
|
total?: number;
|
|
8
8
|
}
|
|
9
|
+
export interface SpiderTransportUsage {
|
|
10
|
+
undici: number;
|
|
11
|
+
curl: number;
|
|
12
|
+
}
|
|
13
|
+
export interface SpiderSecuritySnapshot {
|
|
14
|
+
pages: number;
|
|
15
|
+
blockedPages: number;
|
|
16
|
+
captchaPages: number;
|
|
17
|
+
captchaProviders?: Record<string, number>;
|
|
18
|
+
attempts: number;
|
|
19
|
+
retries: number;
|
|
20
|
+
transportUsage: SpiderTransportUsage;
|
|
21
|
+
avgAttempts: number;
|
|
22
|
+
avgTtfbMs?: number;
|
|
23
|
+
avgTotalMs?: number;
|
|
24
|
+
avgDownloadMs?: number;
|
|
25
|
+
}
|
|
26
|
+
export interface SpiderSerpSummary {
|
|
27
|
+
queriesRequested: number;
|
|
28
|
+
queriesFound: number;
|
|
29
|
+
avgTopPosition?: number | null;
|
|
30
|
+
top3Count: number;
|
|
31
|
+
top10Count: number;
|
|
32
|
+
topOrganicCompetitors?: Array<{
|
|
33
|
+
domain: string;
|
|
34
|
+
matchedKeywords: number;
|
|
35
|
+
totalOutperformedQueries: number;
|
|
36
|
+
}>;
|
|
37
|
+
topPages?: Array<{
|
|
38
|
+
pageUrl: string;
|
|
39
|
+
found: number;
|
|
40
|
+
tracked: number;
|
|
41
|
+
appearanceRate: number;
|
|
42
|
+
}>;
|
|
43
|
+
}
|
|
9
44
|
export interface PageInfo {
|
|
10
45
|
url: string;
|
|
11
46
|
score: number;
|
|
@@ -36,11 +71,39 @@ export interface SpiderTuiState {
|
|
|
36
71
|
duplicateDescriptions: number;
|
|
37
72
|
orphanPages: number;
|
|
38
73
|
pagesWithErrors: number;
|
|
74
|
+
security: SpiderSecuritySnapshot;
|
|
75
|
+
serpSummary?: SpiderSerpSummary;
|
|
76
|
+
antiBotPages: number;
|
|
77
|
+
antiBotBlockedPages: number;
|
|
78
|
+
antiBotCaptchaPages: number;
|
|
79
|
+
antiBotCaptchaProviders: Record<string, number>;
|
|
80
|
+
antiBotAttempts: number;
|
|
81
|
+
antiBotRetries: number;
|
|
82
|
+
antiBotTransport: SpiderTransportUsage;
|
|
83
|
+
antiBotTtfbSamples: number;
|
|
84
|
+
antiBotTotalSamples: number;
|
|
85
|
+
antiBotDownloadSamples: number;
|
|
86
|
+
antiBotTtfbSum: number;
|
|
87
|
+
antiBotTotalSum: number;
|
|
88
|
+
antiBotDownloadSum: number;
|
|
39
89
|
}
|
|
40
90
|
export declare function createSpiderTui(startUrl: string, limit: number, concurrency: number): {
|
|
41
91
|
updateProgress: (crawled: number, queued: number, pending: number, depth: number) => void;
|
|
42
92
|
updateSeo: (url: string, score: number, bytes?: number, timings?: SpiderTimings) => void;
|
|
43
93
|
updateUrl: (url: string) => void;
|
|
94
|
+
updateSecurity: (security?: {
|
|
95
|
+
blocked?: boolean;
|
|
96
|
+
captchaDetected?: boolean;
|
|
97
|
+
captchaProvider?: string;
|
|
98
|
+
attempts?: number;
|
|
99
|
+
retryCount?: number;
|
|
100
|
+
transport?: "undici" | "curl";
|
|
101
|
+
ttfb?: number;
|
|
102
|
+
total?: number;
|
|
103
|
+
download?: number;
|
|
104
|
+
}) => void;
|
|
105
|
+
setSecuritySummary: (summary: SpiderSecuritySnapshot) => void;
|
|
106
|
+
setSerpSummary: (serpSummary: SpiderSerpSummary) => void;
|
|
44
107
|
setComplete: (summary: Partial<SpiderTuiState>) => void;
|
|
45
108
|
setError: (error: string) => void;
|
|
46
109
|
stop: () => void;
|
|
@@ -23,6 +23,34 @@ const initialState = {
|
|
|
23
23
|
duplicateDescriptions: 0,
|
|
24
24
|
orphanPages: 0,
|
|
25
25
|
pagesWithErrors: 0,
|
|
26
|
+
security: {
|
|
27
|
+
pages: 0,
|
|
28
|
+
blockedPages: 0,
|
|
29
|
+
captchaPages: 0,
|
|
30
|
+
attempts: 0,
|
|
31
|
+
retries: 0,
|
|
32
|
+
transportUsage: {
|
|
33
|
+
undici: 0,
|
|
34
|
+
curl: 0,
|
|
35
|
+
},
|
|
36
|
+
avgAttempts: 0,
|
|
37
|
+
},
|
|
38
|
+
antiBotPages: 0,
|
|
39
|
+
antiBotBlockedPages: 0,
|
|
40
|
+
antiBotCaptchaPages: 0,
|
|
41
|
+
antiBotCaptchaProviders: {},
|
|
42
|
+
antiBotAttempts: 0,
|
|
43
|
+
antiBotRetries: 0,
|
|
44
|
+
antiBotTransport: {
|
|
45
|
+
undici: 0,
|
|
46
|
+
curl: 0,
|
|
47
|
+
},
|
|
48
|
+
antiBotTtfbSamples: 0,
|
|
49
|
+
antiBotTotalSamples: 0,
|
|
50
|
+
antiBotDownloadSamples: 0,
|
|
51
|
+
antiBotTtfbSum: 0,
|
|
52
|
+
antiBotTotalSum: 0,
|
|
53
|
+
antiBotDownloadSum: 0,
|
|
26
54
|
};
|
|
27
55
|
const SPARKLINE_SIZE = 30;
|
|
28
56
|
const RECENT_PAGES_SIZE = 15;
|
|
@@ -79,6 +107,23 @@ function renderTimings(s) {
|
|
|
79
107
|
}
|
|
80
108
|
return timingRows;
|
|
81
109
|
}
|
|
110
|
+
function formatSecurityRate(value, total) {
|
|
111
|
+
if (total <= 0)
|
|
112
|
+
return '0%';
|
|
113
|
+
return `${Math.round((value / total) * 100)}%`;
|
|
114
|
+
}
|
|
115
|
+
function safeNumber(value) {
|
|
116
|
+
return typeof value === 'number' && Number.isFinite(value) ? Math.round(value) : undefined;
|
|
117
|
+
}
|
|
118
|
+
function normalizeAttemptCount(value) {
|
|
119
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
120
|
+
return Math.max(1, Math.floor(value));
|
|
121
|
+
}
|
|
122
|
+
return 1;
|
|
123
|
+
}
|
|
124
|
+
function hasTimingSample(value) {
|
|
125
|
+
return typeof value === 'number' && Number.isFinite(value);
|
|
126
|
+
}
|
|
82
127
|
function renderRecentPages(pages, isComplete, totalAnalyzed) {
|
|
83
128
|
if (pages.length === 0 || isComplete)
|
|
84
129
|
return [];
|
|
@@ -174,8 +219,24 @@ export function createSpiderTui(startUrl, limit, concurrency) {
|
|
|
174
219
|
const isComplete = s.status === 'complete';
|
|
175
220
|
const totalToAnalyze = s.analyzed + s.queued + s.pending;
|
|
176
221
|
const analysisPercent = totalToAnalyze > 0 ? Math.min(100, Math.round((s.analyzed / totalToAnalyze) * 100)) : 0;
|
|
222
|
+
const antiBotTotal = s.antiBotPages || s.analyzed || 1;
|
|
223
|
+
const blockedRate = formatSecurityRate(s.antiBotBlockedPages, antiBotTotal);
|
|
224
|
+
const captchaRate = formatSecurityRate(s.antiBotCaptchaPages, antiBotTotal);
|
|
225
|
+
const transportTotal = s.antiBotTransport.undici + s.antiBotTransport.curl;
|
|
226
|
+
const avgAttempts = antiBotTotal > 0 ? (s.antiBotAttempts / antiBotTotal).toFixed(2) : '0.00';
|
|
227
|
+
const avgTtfb = s.antiBotTtfbSamples > 0 ? Math.round(s.antiBotTtfbSum / s.antiBotTtfbSamples) : 'n/a';
|
|
228
|
+
const avgTotal = s.antiBotTotalSamples > 0 ? Math.round(s.antiBotTotalSum / s.antiBotTotalSamples) : 'n/a';
|
|
229
|
+
const avgDownload = s.antiBotDownloadSamples > 0 ? Math.round(s.antiBotDownloadSum / s.antiBotDownloadSamples) : 'n/a';
|
|
230
|
+
const formatTiming = (value) => value === 'n/a' ? 'n/a' : `${value}ms`;
|
|
231
|
+
const curlUsage = transportTotal > 0 ? Math.round((s.antiBotTransport.curl / transportTotal) * 100) : 0;
|
|
232
|
+
const transportLine = transportTotal > 0
|
|
233
|
+
? `curl ${curlUsage}% / undici ${100 - curlUsage}%`
|
|
234
|
+
: 'n/a';
|
|
235
|
+
const captchaProviderSummary = Object.entries(s.antiBotCaptchaProviders)
|
|
236
|
+
.map(([provider, count]) => `${provider}: ${count}`)
|
|
237
|
+
.join(' | ');
|
|
177
238
|
const scoreColor = s.avgScore >= 80 ? 'success' : s.avgScore >= 60 ? 'warning' : 'destructive';
|
|
178
|
-
return Box({ flexDirection: 'column', padding: 1 }, Box({ flexDirection: 'row', marginBottom: 1, height: 1 }, Box({ width: 2 }, StatusIndicator({ status: isComplete ? 'success' : 'info' })), Text({ bold: true }, `SEO Spider: ${startUrl}`)), Box({ flexDirection: 'row', gap: 3, marginBottom: 1 }, Box({ flexDirection: 'column' }, Text({ dim: true }, 'Avg Score'), Text({ color: scoreColor, bold: true }, s.avgScore > 0 ? String(s.avgScore) : '--')), Box({ flexDirection: 'column' }, Text({ dim: true }, 'To Crawl'), Text({ color: (s.queued + s.pending) > 0 ? 'yellow' : 'gray', bold: true }, String(s.queued + s.pending))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Parallel'), Text({ color: 'magenta', bold: true }, String(concurrency))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Depth'), Text({ bold: true }, String(s.depth))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Time'), Text({ bold: true }, `${elapsed}s`)), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Traffic'), Text({ color: 'cyan', bold: true }, formatBytes(s.totalBytes)))), Box({ flexDirection: 'row' }, Box({ width: 8 }, Text({ dim: true }, 'Progress')), ProgressBar({
|
|
239
|
+
return Box({ flexDirection: 'column', padding: 1 }, Box({ flexDirection: 'row', marginBottom: 1, height: 1 }, Box({ width: 2 }, StatusIndicator({ status: isComplete ? 'success' : 'info' })), Text({ bold: true }, `SEO Spider: ${startUrl}`)), Box({ flexDirection: 'row', gap: 1, marginBottom: 1 }, Text({ dim: true }, 'Transport:'), Text({ color: 'magenta', bold: true }, transportLine)), Box({ flexDirection: 'row', gap: 3, marginBottom: 1 }, Box({ flexDirection: 'column' }, Text({ dim: true }, 'Avg Score'), Text({ color: scoreColor, bold: true }, s.avgScore > 0 ? String(s.avgScore) : '--')), Box({ flexDirection: 'column' }, Text({ dim: true }, 'To Crawl'), Text({ color: (s.queued + s.pending) > 0 ? 'yellow' : 'gray', bold: true }, String(s.queued + s.pending))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Parallel'), Text({ color: 'magenta', bold: true }, String(concurrency))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Depth'), Text({ bold: true }, String(s.depth))), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Time'), Text({ bold: true }, `${elapsed}s`)), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Traffic'), Text({ color: 'cyan', bold: true }, formatBytes(s.totalBytes)))), Box({ flexDirection: 'row', gap: 3, marginBottom: 1 }, Box({ flexDirection: 'column' }, Text({ dim: true }, 'Anti-bot'), Text({ color: s.antiBotBlockedPages > 0 ? 'warning' : 'success', bold: true }, `${s.antiBotBlockedPages}/${antiBotTotal} blocked (${blockedRate})`)), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Captcha'), Text({ color: s.antiBotCaptchaPages > 0 ? 'warning' : 'success', bold: true }, `${s.antiBotCaptchaPages} (${captchaRate})`)), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Captcha Providers'), Text({ color: 'blue', bold: true }, captchaProviderSummary || 'n/a')), Box({ flexDirection: 'column' }, Text({ dim: true }, 'Attempts'), Text({ color: 'blue', bold: true }, `avg ${avgAttempts}`))), Box({ flexDirection: 'row', gap: 3, marginBottom: 1 }, Box({ flexDirection: 'column' }, Text({ dim: true }, 'Avg timing'), Text({ color: 'cyan', bold: true }, `ttfb ${formatTiming(avgTtfb)} · total ${formatTiming(avgTotal)}`), Text({ color: 'cyan', bold: true }, `download ${formatTiming(avgDownload)}`))), Box({ flexDirection: 'row' }, Box({ width: 8 }, Text({ dim: true }, 'Progress')), ProgressBar({
|
|
179
240
|
value: analysisPercent,
|
|
180
241
|
width: 25,
|
|
181
242
|
showPercentage: true,
|
|
@@ -195,7 +256,15 @@ export function createSpiderTui(startUrl, limit, concurrency) {
|
|
|
195
256
|
width: 25,
|
|
196
257
|
color: 'cyan',
|
|
197
258
|
}), Box({ marginLeft: 1 }, Text({ color: 'cyan' }, `${last(s.pageSizes) ?? 0}KB`))),
|
|
198
|
-
] : []), ...renderTimings(s), ...renderRecentPages(s.recentPages, isComplete, s.analyzed), ...(
|
|
259
|
+
] : []), ...renderTimings(s), ...renderRecentPages(s.recentPages, isComplete, s.analyzed), ...(s.serpSummary ? [
|
|
260
|
+
Box({ flexDirection: 'column', marginTop: 1 }, Divider({}), Box({ flexDirection: 'row', marginTop: 1, height: 1 }, Text({ bold: true }, 'SERP campaign'), Box({ width: 2 }, Text({}, '')), Text({ color: 'green' }, `queries ${s.serpSummary.queriesFound}/${s.serpSummary.queriesRequested}`), Box({ width: 2 }, Text({}, '')), Text({ color: 'blue' }, `top3 ${s.serpSummary.top3Count}`), Box({ width: 2 }, Text({}, '')), Text({ color: 'cyan' }, `top10 ${s.serpSummary.top10Count}`), s.serpSummary.avgTopPosition !== undefined && s.serpSummary.avgTopPosition !== null
|
|
261
|
+
? Box({ width: 2 }, Text({ color: 'yellow' }, `avg pos ${Math.round(s.serpSummary.avgTopPosition)}`))
|
|
262
|
+
: null), s.serpSummary.topOrganicCompetitors && s.serpSummary.topOrganicCompetitors.length > 0
|
|
263
|
+
? Box({ marginTop: 1 }, Text({ dim: true }, 'Top competitors'))
|
|
264
|
+
: null, ...(s.serpSummary.topOrganicCompetitors || []).slice(0, 3).map((item) => Box({ flexDirection: 'row' }, Text({ dim: true }, `${item.domain}`), Text({ dim: true }, ` wins ${item.totalOutperformedQueries}`), Text({ dim: true }, ` matched ${item.matchedKeywords}`))), s.serpSummary.topPages && s.serpSummary.topPages.length > 0
|
|
265
|
+
? Box({ marginTop: 1 }, Text({ dim: true }, 'Pages outranking baseline'))
|
|
266
|
+
: null, ...(s.serpSummary.topPages || []).slice(0, 3).map((item) => Box({ flexDirection: 'row' }, Text({ dim: true }, `${item.pageUrl}`), Text({ dim: true }, `${item.found}/${item.tracked}`), Text({ dim: true }, `${Math.round(item.appearanceRate)}%`)))),
|
|
267
|
+
] : []), ...(isComplete ? [
|
|
199
268
|
Box({ flexDirection: 'column', marginTop: 1 }, Divider({}), Box({ flexDirection: 'row', gap: 2, marginTop: 1, height: 1 }, ...[
|
|
200
269
|
s.duplicateTitles > 0 ? Badge({ label: `${s.duplicateTitles} dup titles`, color: 'destructive' }) : null,
|
|
201
270
|
s.duplicateDescriptions > 0 ? Badge({ label: `${s.duplicateDescriptions} dup desc`, color: 'warning' }) : null,
|
|
@@ -209,6 +278,55 @@ export function createSpiderTui(startUrl, limit, concurrency) {
|
|
|
209
278
|
updateProgress,
|
|
210
279
|
updateSeo,
|
|
211
280
|
updateUrl,
|
|
281
|
+
updateSecurity: (security) => {
|
|
282
|
+
setState((s) => ({
|
|
283
|
+
...s,
|
|
284
|
+
antiBotPages: s.antiBotPages + 1,
|
|
285
|
+
antiBotBlockedPages: s.antiBotBlockedPages + (security?.blocked ? 1 : 0),
|
|
286
|
+
antiBotCaptchaPages: s.antiBotCaptchaPages + (security?.captchaDetected ? 1 : 0),
|
|
287
|
+
antiBotCaptchaProviders: security?.captchaDetected ? {
|
|
288
|
+
...s.antiBotCaptchaProviders,
|
|
289
|
+
[(security.captchaProvider || 'unknown').trim().toLowerCase()]: (s.antiBotCaptchaProviders[(security.captchaProvider || 'unknown').trim().toLowerCase()] || 0) + 1,
|
|
290
|
+
} : s.antiBotCaptchaProviders,
|
|
291
|
+
antiBotAttempts: s.antiBotAttempts + normalizeAttemptCount(security?.attempts),
|
|
292
|
+
antiBotRetries: s.antiBotRetries + (security?.retryCount ?? 0),
|
|
293
|
+
antiBotTransport: {
|
|
294
|
+
undici: s.antiBotTransport.undici + (security?.transport === 'undici' ? 1 : 0),
|
|
295
|
+
curl: s.antiBotTransport.curl + (security?.transport === 'curl' ? 1 : 0),
|
|
296
|
+
},
|
|
297
|
+
antiBotTtfbSamples: s.antiBotTtfbSamples + (hasTimingSample(security?.ttfb) ? 1 : 0),
|
|
298
|
+
antiBotTotalSamples: s.antiBotTotalSamples + (hasTimingSample(security?.total) ? 1 : 0),
|
|
299
|
+
antiBotDownloadSamples: s.antiBotDownloadSamples + (hasTimingSample(security?.download) ? 1 : 0),
|
|
300
|
+
antiBotTtfbSum: s.antiBotTtfbSum + (safeNumber(security?.ttfb) ?? 0),
|
|
301
|
+
antiBotTotalSum: s.antiBotTotalSum + (safeNumber(security?.total) ?? 0),
|
|
302
|
+
antiBotDownloadSum: s.antiBotDownloadSum + (safeNumber(security?.download) ?? 0),
|
|
303
|
+
}));
|
|
304
|
+
},
|
|
305
|
+
setSecuritySummary: (summary) => {
|
|
306
|
+
setState((s) => ({
|
|
307
|
+
...s,
|
|
308
|
+
security: summary,
|
|
309
|
+
antiBotPages: summary.pages,
|
|
310
|
+
antiBotBlockedPages: summary.blockedPages,
|
|
311
|
+
antiBotCaptchaPages: summary.captchaPages,
|
|
312
|
+
antiBotAttempts: summary.attempts,
|
|
313
|
+
antiBotRetries: summary.retries,
|
|
314
|
+
antiBotTransport: {
|
|
315
|
+
undici: summary.transportUsage.undici,
|
|
316
|
+
curl: summary.transportUsage.curl,
|
|
317
|
+
},
|
|
318
|
+
antiBotCaptchaProviders: summary.captchaProviders || {},
|
|
319
|
+
antiBotTtfbSamples: summary.avgTtfbMs === undefined ? 0 : summary.pages,
|
|
320
|
+
antiBotTtfbSum: summary.avgTtfbMs === undefined ? 0 : Math.round(summary.avgTtfbMs * summary.pages),
|
|
321
|
+
antiBotTotalSamples: summary.avgTotalMs === undefined ? 0 : summary.pages,
|
|
322
|
+
antiBotTotalSum: summary.avgTotalMs === undefined ? 0 : Math.round(summary.avgTotalMs * summary.pages),
|
|
323
|
+
antiBotDownloadSamples: summary.avgDownloadMs === undefined ? 0 : summary.pages,
|
|
324
|
+
antiBotDownloadSum: summary.avgDownloadMs === undefined ? 0 : Math.round(summary.avgDownloadMs * summary.pages),
|
|
325
|
+
}));
|
|
326
|
+
},
|
|
327
|
+
setSerpSummary: (serpSummary) => {
|
|
328
|
+
setState((s) => ({ ...s, serpSummary }));
|
|
329
|
+
},
|
|
212
330
|
setComplete,
|
|
213
331
|
setError,
|
|
214
332
|
stop: () => {
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ClientOptions, Method, RequestOptions } from '../types/index.js';
|
|
2
|
+
export type CliRequestHeaders = Record<string, string>;
|
|
3
|
+
export type CliRequestOptions = Omit<RequestOptions, 'method' | 'headers' | 'body'> & {
|
|
4
|
+
method: Method;
|
|
5
|
+
url: string;
|
|
6
|
+
headers?: CliRequestHeaders;
|
|
7
|
+
body?: RequestOptions['json'] | RequestOptions['body'];
|
|
8
|
+
verbose?: boolean;
|
|
9
|
+
quiet?: boolean;
|
|
10
|
+
output?: string;
|
|
11
|
+
clientOptions?: ClientOptions;
|
|
12
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { SpiderTransport } from '../../scrape/spider.js';
|
|
2
|
+
export type { SpiderTransport };
|
|
3
|
+
type OptionBag = Record<string, unknown>;
|
|
4
|
+
export type SpiderFocusMode = 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
|
|
5
|
+
export declare function getOptionValue(ctxOptions: OptionBag, ...keys: string[]): unknown;
|
|
6
|
+
export declare function toOptionString(value: unknown): string | undefined;
|
|
7
|
+
export declare function toBoolean(value: unknown, fallback?: boolean): boolean;
|
|
8
|
+
export declare function toNonNegativeInt(value: unknown, fallback?: number): number;
|
|
9
|
+
export declare function toSpiderTransport(value: unknown): SpiderTransport | undefined;
|
|
10
|
+
export declare function toSpiderFocusMode(value: unknown): SpiderFocusMode;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
const SPIDER_FOCUS_MODES = ['all', 'links', 'duplicates', 'security', 'ai', 'resources'];
|
|
2
|
+
export function getOptionValue(ctxOptions, ...keys) {
|
|
3
|
+
for (const key of keys) {
|
|
4
|
+
if (Object.prototype.hasOwnProperty.call(ctxOptions, key)) {
|
|
5
|
+
return ctxOptions[key];
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
return undefined;
|
|
9
|
+
}
|
|
10
|
+
export function toOptionString(value) {
|
|
11
|
+
if (typeof value === 'string') {
|
|
12
|
+
const cleaned = value.trim();
|
|
13
|
+
return cleaned.length > 0 ? cleaned : undefined;
|
|
14
|
+
}
|
|
15
|
+
if (typeof value === 'number')
|
|
16
|
+
return Number.isFinite(value) ? String(value) : undefined;
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
|
19
|
+
export function toBoolean(value, fallback = false) {
|
|
20
|
+
if (typeof value === 'boolean') {
|
|
21
|
+
return value;
|
|
22
|
+
}
|
|
23
|
+
if (typeof value === 'number') {
|
|
24
|
+
return value !== 0;
|
|
25
|
+
}
|
|
26
|
+
if (typeof value === 'string') {
|
|
27
|
+
const normalized = value.trim().toLowerCase();
|
|
28
|
+
if (normalized === '' || normalized === 'default') {
|
|
29
|
+
return fallback;
|
|
30
|
+
}
|
|
31
|
+
if (['1', 'true', 'on', 'yes'].includes(normalized)) {
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
if (['0', 'false', 'off', 'no', 'none'].includes(normalized)) {
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
return fallback;
|
|
38
|
+
}
|
|
39
|
+
if (value && typeof value === 'object' && 'default' in value) {
|
|
40
|
+
return toBoolean(value.default, fallback);
|
|
41
|
+
}
|
|
42
|
+
return fallback;
|
|
43
|
+
}
|
|
44
|
+
export function toNonNegativeInt(value, fallback = 0) {
|
|
45
|
+
const parsed = typeof value === 'number'
|
|
46
|
+
? value
|
|
47
|
+
: typeof value === 'string'
|
|
48
|
+
? Number(value.trim())
|
|
49
|
+
: fallback;
|
|
50
|
+
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallback;
|
|
51
|
+
}
|
|
52
|
+
export function toSpiderTransport(value) {
|
|
53
|
+
const raw = toOptionString(value)?.toLowerCase();
|
|
54
|
+
if (raw === 'auto' || raw === 'undici' || raw === 'curl')
|
|
55
|
+
return raw;
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
|
58
|
+
export function toSpiderFocusMode(value) {
|
|
59
|
+
const raw = toOptionString(value)?.toLowerCase();
|
|
60
|
+
return raw && SPIDER_FOCUS_MODES.includes(raw)
|
|
61
|
+
? raw
|
|
62
|
+
: 'all';
|
|
63
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export type ScoreColor = (value: string) => string;
|
|
2
|
+
export type ScorePalette = {
|
|
3
|
+
green: ScoreColor;
|
|
4
|
+
yellow: ScoreColor;
|
|
5
|
+
red: ScoreColor;
|
|
6
|
+
};
|
|
7
|
+
export type ScoreColorThresholds = {
|
|
8
|
+
high: number;
|
|
9
|
+
medium: number;
|
|
10
|
+
};
|
|
11
|
+
export declare function getScoreColor(score: number, colors: ScorePalette, thresholds?: ScoreColorThresholds): ScoreColor;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
const DEFAULT_SCORE_THRESHOLDS = {
|
|
2
|
+
high: 80,
|
|
3
|
+
medium: 60,
|
|
4
|
+
};
|
|
5
|
+
export function getScoreColor(score, colors, thresholds = DEFAULT_SCORE_THRESHOLDS) {
|
|
6
|
+
if (score >= thresholds.high)
|
|
7
|
+
return colors.green;
|
|
8
|
+
if (score >= thresholds.medium)
|
|
9
|
+
return colors.yellow;
|
|
10
|
+
return colors.red;
|
|
11
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { SpiderSerpOptionConfig } from './serp-config.js';
|
|
2
|
+
import type { SeoCrawlerSerpRun, SerpSeedPage } from '../handlers/seo.js';
|
|
3
|
+
export type SpiderSerpCampaignSeed = {
|
|
4
|
+
keyword: string;
|
|
5
|
+
sourcePage?: string;
|
|
6
|
+
sourceWeight: number;
|
|
7
|
+
};
|
|
8
|
+
export interface SpiderSerpCampaignPayload {
|
|
9
|
+
summary: {
|
|
10
|
+
queriesRequested: number;
|
|
11
|
+
queriesExecuted: number;
|
|
12
|
+
queriesFound: number;
|
|
13
|
+
avgTopPosition: number | null;
|
|
14
|
+
top3Count: number;
|
|
15
|
+
top10Count: number;
|
|
16
|
+
topOrganicCompetitors: Array<{
|
|
17
|
+
domain: string;
|
|
18
|
+
matchedKeywords: number;
|
|
19
|
+
totalOutperformedQueries?: number;
|
|
20
|
+
organicQueries?: number;
|
|
21
|
+
paidQueries?: number;
|
|
22
|
+
avgOutperformedGap?: number | null;
|
|
23
|
+
}>;
|
|
24
|
+
};
|
|
25
|
+
campaign: {
|
|
26
|
+
active: boolean;
|
|
27
|
+
confidence: 'high' | 'medium' | 'low';
|
|
28
|
+
evidence: string[];
|
|
29
|
+
};
|
|
30
|
+
results: Array<{
|
|
31
|
+
keyword: string;
|
|
32
|
+
found: boolean;
|
|
33
|
+
position: number | null;
|
|
34
|
+
targetUrl?: string;
|
|
35
|
+
searchUrl: string;
|
|
36
|
+
}>;
|
|
37
|
+
pageComparison: Array<{
|
|
38
|
+
pageUrl: string;
|
|
39
|
+
tracked: number;
|
|
40
|
+
found: number;
|
|
41
|
+
appearanceRate: string;
|
|
42
|
+
avgPosition: string;
|
|
43
|
+
top3: number;
|
|
44
|
+
top10: number;
|
|
45
|
+
}>;
|
|
46
|
+
seedPlan: {
|
|
47
|
+
short: SpiderSerpCampaignSeed[];
|
|
48
|
+
longTail: SpiderSerpCampaignSeed[];
|
|
49
|
+
ordered: SpiderSerpCampaignSeed[];
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
export declare function runSpiderKeywordCampaign(targetUrl: string, pages: SerpSeedPage[], config: SpiderSerpOptionConfig): Promise<SeoCrawlerSerpRun | undefined>;
|
|
53
|
+
export declare function toSerpPayload(campaign: SeoCrawlerSerpRun): SpiderSerpCampaignPayload;
|