recker 1.0.79 → 1.0.80-next.70ea84d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/browser/mini.d.ts +2 -2
- package/dist/browser/browser/recker-mini.d.ts +8 -8
- package/dist/browser/browser/recker.d.ts +11 -8
- package/dist/browser/browser/recker.js +54 -6
- package/dist/browser/core/client.d.ts +15 -10
- package/dist/browser/core/client.js +54 -38
- package/dist/browser/index.iife.min.js +129 -130
- package/dist/browser/index.min.js +129 -130
- package/dist/browser/index.mini.iife.js +5697 -636
- package/dist/browser/index.mini.iife.min.js +47 -48
- package/dist/browser/index.mini.min.js +47 -48
- package/dist/browser/index.mini.umd.js +5697 -636
- package/dist/browser/index.mini.umd.min.js +47 -48
- package/dist/browser/index.umd.min.js +129 -130
- package/dist/browser/mini.d.ts +2 -2
- package/dist/browser/plugins/proxy-rotator.d.ts +2 -2
- package/dist/browser/plugins/proxy-rotator.js +6 -28
- package/dist/browser/recker-mini.d.ts +8 -8
- package/dist/browser/recker.d.ts +11 -8
- package/dist/browser/recker.js +54 -6
- package/dist/browser/scrape/document.js +2 -2
- package/dist/browser/scrape/element.js +7 -1
- package/dist/browser/scrape/parser/nodes/html.js +1 -1
- package/dist/browser/scrape/spider.d.ts +52 -0
- package/dist/browser/scrape/spider.js +620 -38
- package/dist/browser/scrape/types.d.ts +2 -0
- package/dist/browser/search/google.d.ts +26 -1
- package/dist/browser/search/google.js +427 -45
- package/dist/browser/seo/analyzer.d.ts +1 -0
- package/dist/browser/seo/analyzer.js +144 -1
- package/dist/browser/seo/index.d.ts +1 -1
- package/dist/browser/seo/keyword-campaign-analyzer.d.ts +2 -0
- package/dist/browser/seo/keyword-campaign-analyzer.js +538 -0
- package/dist/browser/seo/keyword-campaign-seed-advanced.d.ts +17 -0
- package/dist/browser/seo/keyword-campaign-seed-advanced.js +269 -0
- package/dist/browser/seo/keyword-campaign-seed-core.d.ts +29 -0
- package/dist/browser/seo/keyword-campaign-seed-core.js +525 -0
- package/dist/browser/seo/keyword-campaign-shared.d.ts +165 -0
- package/dist/browser/seo/keyword-campaign-shared.js +59 -0
- package/dist/browser/seo/keyword-campaign.d.ts +4 -107
- package/dist/browser/seo/keyword-campaign.js +2 -380
- package/dist/browser/seo/keywords.js +5 -22
- package/dist/browser/seo/types.d.ts +19 -0
- package/dist/browser/transport/curl.d.ts +5 -1
- package/dist/browser/transport/curl.js +207 -50
- package/dist/browser/transport/undici.d.ts +4 -3
- package/dist/browser/transport/undici.js +123 -49
- package/dist/browser/types/index.d.ts +9 -3
- package/dist/browser/utils/binary-manager.js +26 -3
- package/dist/browser/utils/block-detector.d.ts +8 -0
- package/dist/browser/utils/block-detector.js +542 -7
- package/dist/cli/commands/hls-runner.js +5 -4
- package/dist/cli/commands/live-runner.js +5 -4
- package/dist/cli/commands/loadtest-runner.js +3 -2
- package/dist/cli/commands/search.d.ts +2 -0
- package/dist/cli/commands/search.js +105 -0
- package/dist/cli/commands/seo-runner.js +9 -7
- package/dist/cli/commands/seo.js +140 -1
- package/dist/cli/commands/serve.js +75 -131
- package/dist/cli/commands/server-runner.js +59 -82
- package/dist/cli/commands/spider-runner.d.ts +37 -1
- package/dist/cli/commands/spider-runner.js +134 -10
- package/dist/cli/commands/spider.d.ts +18 -1
- package/dist/cli/commands/spider.js +457 -27
- package/dist/cli/events/handlers/cli.js +30 -1
- package/dist/cli/events/handlers/tui.js +26 -0
- package/dist/cli/events/types.d.ts +27 -0
- package/dist/cli/handler.d.ts +2 -12
- package/dist/cli/handler.js +20 -15
- package/dist/cli/handlers/protocols.js +39 -12
- package/dist/cli/handlers/search.d.ts +2 -0
- package/dist/cli/handlers/search.js +171 -0
- package/dist/cli/handlers/seo-analyze.d.ts +1 -0
- package/dist/cli/handlers/seo-analyze.js +666 -0
- package/dist/cli/handlers/seo-robots.d.ts +1 -0
- package/dist/cli/handlers/seo-robots.js +76 -0
- package/dist/cli/handlers/seo-serp.d.ts +54 -0
- package/dist/cli/handlers/seo-serp.js +243 -0
- package/dist/cli/handlers/seo-sitemap.d.ts +1 -0
- package/dist/cli/handlers/seo-sitemap.js +55 -0
- package/dist/cli/handlers/seo-spider.d.ts +1 -0
- package/dist/cli/handlers/seo-spider.js +334 -0
- package/dist/cli/handlers/seo.d.ts +8 -4
- package/dist/cli/handlers/seo.js +294 -442
- package/dist/cli/handlers/spider.js +94 -17
- package/dist/cli/handlers/streaming.js +5 -1
- package/dist/cli/index.js +11 -2
- package/dist/cli/presets.d.ts +1 -1
- package/dist/cli/presets.js +15 -4
- package/dist/cli/tui/app.js +6 -1
- package/dist/cli/tui/components/rich-response.d.ts +72 -0
- package/dist/cli/tui/components/rich-response.js +117 -0
- package/dist/cli/tui/executor-commands/background.js +30 -24
- package/dist/cli/tui/executor-commands/testing.js +3 -2
- package/dist/cli/tui/hooks/useDomains.d.ts +17 -0
- package/dist/cli/tui/hooks/useHelp.js +15 -2
- package/dist/cli/tui/job-manager.d.ts +4 -4
- package/dist/cli/tui/job-manager.js +5 -1
- package/dist/cli/tui/spider-tui.d.ts +63 -0
- package/dist/cli/tui/spider-tui.js +120 -2
- package/dist/cli/types.d.ts +12 -0
- package/dist/cli/types.js +1 -0
- package/dist/cli/utils/option-helpers.d.ts +10 -0
- package/dist/cli/utils/option-helpers.js +63 -0
- package/dist/cli/utils/score-color.d.ts +11 -0
- package/dist/cli/utils/score-color.js +11 -0
- package/dist/cli/utils/serp-campaign.d.ts +53 -0
- package/dist/cli/utils/serp-campaign.js +53 -0
- package/dist/cli/utils/serp-config.d.ts +26 -0
- package/dist/cli/utils/serp-config.js +125 -0
- package/dist/core/client.d.ts +15 -10
- package/dist/core/client.js +54 -38
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp/cli.js +35 -34
- package/dist/mcp/client.d.ts +2 -2
- package/dist/mcp/client.js +20 -2
- package/dist/mcp/contract.d.ts +1 -1
- package/dist/mcp/profiles.js +5 -1
- package/dist/mcp/prompts/index.js +8 -4
- package/dist/mcp/resources/index.js +46 -23
- package/dist/mcp/server.js +9 -6
- package/dist/mcp/tools/protocols.js +9 -2
- package/dist/mcp/tools/registry.js +13 -3
- package/dist/mcp/tools/seo.js +427 -2
- package/dist/mcp/types.d.ts +5 -1
- package/dist/plugins/proxy-rotator.d.ts +2 -2
- package/dist/plugins/proxy-rotator.js +6 -28
- package/dist/raffel/client.d.ts +38 -0
- package/dist/raffel/client.js +282 -0
- package/dist/raffel/index.d.ts +2 -0
- package/dist/raffel/index.js +2 -0
- package/dist/raffel/types.d.ts +40 -0
- package/dist/raffel/types.js +14 -0
- package/dist/recker.d.ts +13 -7
- package/dist/recker.js +58 -6
- package/dist/scrape/document.js +2 -2
- package/dist/scrape/element.js +7 -1
- package/dist/scrape/parser/nodes/html.js +1 -1
- package/dist/scrape/spider.d.ts +52 -0
- package/dist/scrape/spider.js +620 -38
- package/dist/scrape/types.d.ts +2 -0
- package/dist/search/google.d.ts +26 -1
- package/dist/search/google.js +427 -45
- package/dist/search/index.d.ts +1 -1
- package/dist/seo/analyzer.d.ts +1 -0
- package/dist/seo/analyzer.js +144 -1
- package/dist/seo/index.d.ts +1 -1
- package/dist/seo/keyword-campaign-analyzer.d.ts +2 -0
- package/dist/seo/keyword-campaign-analyzer.js +538 -0
- package/dist/seo/keyword-campaign-seed-advanced.d.ts +17 -0
- package/dist/seo/keyword-campaign-seed-advanced.js +269 -0
- package/dist/seo/keyword-campaign-seed-core.d.ts +29 -0
- package/dist/seo/keyword-campaign-seed-core.js +525 -0
- package/dist/seo/keyword-campaign-shared.d.ts +165 -0
- package/dist/seo/keyword-campaign-shared.js +59 -0
- package/dist/seo/keyword-campaign.d.ts +4 -107
- package/dist/seo/keyword-campaign.js +2 -380
- package/dist/seo/keywords.js +5 -22
- package/dist/seo/types.d.ts +19 -0
- package/dist/template/index.d.ts +1 -1
- package/dist/template/types.d.ts +0 -2
- package/dist/testing/index.d.ts +1 -22
- package/dist/testing/index.js +1 -11
- package/dist/transport/curl.d.ts +5 -1
- package/dist/transport/curl.js +207 -50
- package/dist/transport/undici.d.ts +4 -3
- package/dist/transport/undici.js +123 -49
- package/dist/types/index.d.ts +9 -3
- package/dist/utils/binary-manager.js +26 -3
- package/dist/utils/block-detector.d.ts +8 -0
- package/dist/utils/block-detector.js +542 -7
- package/dist/version.js +1 -1
- package/package.json +12 -1
- package/dist/testing/mock-dns-server.d.ts +0 -69
- package/dist/testing/mock-dns-server.js +0 -269
- package/dist/testing/mock-ftp-server.d.ts +0 -89
- package/dist/testing/mock-ftp-server.js +0 -562
- package/dist/testing/mock-hls-server.d.ts +0 -80
- package/dist/testing/mock-hls-server.js +0 -381
- package/dist/testing/mock-http-server.d.ts +0 -124
- package/dist/testing/mock-http-server.js +0 -343
- package/dist/testing/mock-proxy-server.d.ts +0 -108
- package/dist/testing/mock-proxy-server.js +0 -615
- package/dist/testing/mock-sse-server.d.ts +0 -76
- package/dist/testing/mock-sse-server.js +0 -291
- package/dist/testing/mock-telnet-server.d.ts +0 -59
- package/dist/testing/mock-telnet-server.js +0 -274
- package/dist/testing/mock-udp-server.d.ts +0 -43
- package/dist/testing/mock-udp-server.js +0 -188
- package/dist/testing/mock-websocket-server.d.ts +0 -76
- package/dist/testing/mock-websocket-server.js +0 -334
- package/dist/testing/mock-whois-server.d.ts +0 -56
- package/dist/testing/mock-whois-server.js +0 -234
- package/dist/testing/proxy-certs.d.ts +0 -19
- package/dist/testing/proxy-certs.js +0 -208
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { withHandler, } from '../output.js';
|
|
2
|
+
import { normalizeUrl } from './seo-serp.js';
|
|
3
|
+
export const robotsHandler = withHandler({ loading: true }, async (ctx, out, extCtx) => {
|
|
4
|
+
let url = ctx.result.positional.url;
|
|
5
|
+
if (!url) {
|
|
6
|
+
const base = extCtx?.baseUrl?.();
|
|
7
|
+
if (base) {
|
|
8
|
+
url = base;
|
|
9
|
+
}
|
|
10
|
+
else {
|
|
11
|
+
out.error('Usage: robots <url> or set base URL first');
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
url = normalizeUrl(url);
|
|
16
|
+
const robotsUrl = new URL('/robots.txt', url).toString();
|
|
17
|
+
let response;
|
|
18
|
+
if (extCtx?.client) {
|
|
19
|
+
response = await extCtx.client.get(robotsUrl);
|
|
20
|
+
}
|
|
21
|
+
else {
|
|
22
|
+
response = await fetch(robotsUrl);
|
|
23
|
+
}
|
|
24
|
+
const text = await response.text();
|
|
25
|
+
if (response.status === 404) {
|
|
26
|
+
out.warn(`No robots.txt found at ${robotsUrl}`);
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
const lines = text.split('\n');
|
|
30
|
+
const directives = [];
|
|
31
|
+
let currentAgent = '*';
|
|
32
|
+
for (const line of lines) {
|
|
33
|
+
const trimmed = line.trim();
|
|
34
|
+
if (trimmed.toLowerCase().startsWith('user-agent:')) {
|
|
35
|
+
currentAgent = trimmed.slice(11).trim();
|
|
36
|
+
}
|
|
37
|
+
else if (trimmed.toLowerCase().startsWith('disallow:')) {
|
|
38
|
+
directives.push({ agent: currentAgent, type: 'disallow', path: trimmed.slice(9).trim() });
|
|
39
|
+
}
|
|
40
|
+
else if (trimmed.toLowerCase().startsWith('allow:')) {
|
|
41
|
+
directives.push({ agent: currentAgent, type: 'allow', path: trimmed.slice(6).trim() });
|
|
42
|
+
}
|
|
43
|
+
else if (trimmed.toLowerCase().startsWith('sitemap:')) {
|
|
44
|
+
directives.push({ type: 'sitemap', url: trimmed.slice(8).trim() });
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const result = {
|
|
48
|
+
url: robotsUrl,
|
|
49
|
+
rules: directives.length,
|
|
50
|
+
sitemaps: directives.filter(d => d.type === 'sitemap').map(d => d.url),
|
|
51
|
+
disallowed: directives.filter(d => d.type === 'disallow').slice(0, 10).map(d => d.path),
|
|
52
|
+
};
|
|
53
|
+
if (extCtx) {
|
|
54
|
+
out.response(result, { responseType: 'robots' });
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
out.title('robots.txt Analysis', '🤖');
|
|
58
|
+
out.keyValue({
|
|
59
|
+
URL: robotsUrl,
|
|
60
|
+
Rules: directives.length,
|
|
61
|
+
});
|
|
62
|
+
out.blank();
|
|
63
|
+
if (result.sitemaps.length > 0) {
|
|
64
|
+
out.subtitle('Sitemaps');
|
|
65
|
+
out.list(result.sitemaps);
|
|
66
|
+
out.blank();
|
|
67
|
+
}
|
|
68
|
+
if (result.disallowed.length > 0) {
|
|
69
|
+
out.subtitle('Disallowed Paths');
|
|
70
|
+
for (const p of result.disallowed) {
|
|
71
|
+
out.status('error', p);
|
|
72
|
+
}
|
|
73
|
+
out.blank();
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
});
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { analyzeKeywordCampaign } from '../../seo/index.js';
|
|
2
|
+
import type { SeoReport } from '../../seo/types.js';
|
|
3
|
+
import { parseSpiderSerpConfig, type SpiderSerpOptionConfig } from '../utils/serp-config.js';
|
|
4
|
+
export declare function normalizeUrl(url: string): string;
|
|
5
|
+
export declare function extractDomain(url: string): string;
|
|
6
|
+
export type SerpSeedPage = {
|
|
7
|
+
url: string;
|
|
8
|
+
seoReport?: SeoReport;
|
|
9
|
+
};
|
|
10
|
+
export type SeoCrawlerSerpConfig = SpiderSerpOptionConfig;
|
|
11
|
+
type SeoCrawlerSerpSeed = {
|
|
12
|
+
keyword: string;
|
|
13
|
+
sourcePage?: string;
|
|
14
|
+
sourceWeight: number;
|
|
15
|
+
};
|
|
16
|
+
export type SeoCrawlerSerpPlan = {
|
|
17
|
+
short: SeoCrawlerSerpSeed[];
|
|
18
|
+
longTail: SeoCrawlerSerpSeed[];
|
|
19
|
+
ordered: SeoCrawlerSerpSeed[];
|
|
20
|
+
};
|
|
21
|
+
export type SeoCrawlerSerpRun = {
|
|
22
|
+
campaign: Awaited<ReturnType<typeof analyzeKeywordCampaign>>;
|
|
23
|
+
plan: SeoCrawlerSerpPlan;
|
|
24
|
+
};
|
|
25
|
+
export declare function buildSerpCampaignSeeds(pages: SerpSeedPage[], topKeywordsPerPage: number, maxSeeds: number): SeoCrawlerSerpPlan;
|
|
26
|
+
export declare function runCrawlerSerpCampaign(targetUrl: string, pages: SerpSeedPage[], config: SeoCrawlerSerpConfig): Promise<SeoCrawlerSerpRun | undefined>;
|
|
27
|
+
export declare function formatSerpSummaryRows(campaign: Awaited<ReturnType<typeof analyzeKeywordCampaign>>): Array<{
|
|
28
|
+
keyword: string;
|
|
29
|
+
found: boolean;
|
|
30
|
+
blocked?: boolean;
|
|
31
|
+
blockReason?: string;
|
|
32
|
+
position: number | null;
|
|
33
|
+
targetUrl?: string;
|
|
34
|
+
searchUrl: string;
|
|
35
|
+
}>;
|
|
36
|
+
export declare function getSerpComparisonRows(campaign: Awaited<ReturnType<typeof analyzeKeywordCampaign>>): Array<{
|
|
37
|
+
pageUrl: string;
|
|
38
|
+
tracked: number;
|
|
39
|
+
found: number;
|
|
40
|
+
appearanceRate: string;
|
|
41
|
+
avgPosition: string;
|
|
42
|
+
top3: number;
|
|
43
|
+
top10: number;
|
|
44
|
+
}>;
|
|
45
|
+
export declare function getSerpCompetitorRows(competitors: Awaited<ReturnType<typeof analyzeKeywordCampaign>>['summary']['topOrganicCompetitors']): Array<{
|
|
46
|
+
domain: string;
|
|
47
|
+
queries: number;
|
|
48
|
+
matchedKeywords: number;
|
|
49
|
+
wins: number;
|
|
50
|
+
avgGap: string;
|
|
51
|
+
bestPages: string;
|
|
52
|
+
}>;
|
|
53
|
+
export declare function formatKeywordList(keywords: string[], maxItems: number): string;
|
|
54
|
+
export { parseSpiderSerpConfig };
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { analyzeKeywordCampaign, extractKeywordCampaignSeedsFromReport, } from '../../seo/index.js';
|
|
2
|
+
import { normalizeKeyword } from '../../seo/keyword-campaign-shared.js';
|
|
3
|
+
import { parseSpiderSerpConfig } from '../utils/serp-config.js';
|
|
4
|
+
import { inferSearchLocaleFromUrl } from '../../search/google.js';
|
|
5
|
+
export function normalizeUrl(url) {
|
|
6
|
+
return url.startsWith('http') ? url : `https://${url}`;
|
|
7
|
+
}
|
|
8
|
+
export function extractDomain(url) {
|
|
9
|
+
try {
|
|
10
|
+
return new URL(url).hostname;
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return url.replace(/^https?:\/\//, '').split('/')[0];
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
const SERP_MIN_SHORT_KEYWORD_LENGTH = 5;
|
|
17
|
+
const SERP_MIN_LONG_TERM_LENGTH = SERP_MIN_SHORT_KEYWORD_LENGTH - 1;
|
|
18
|
+
function normalizeSerpSeedPermutationKey(value) {
|
|
19
|
+
const normalized = normalizeKeyword(value);
|
|
20
|
+
const tokens = normalized.split(/\s+/).filter((token) => token.length > 0);
|
|
21
|
+
if (tokens.length === 0)
|
|
22
|
+
return '';
|
|
23
|
+
if (tokens.length === 1)
|
|
24
|
+
return tokens[0];
|
|
25
|
+
return [...tokens].sort().join(' ');
|
|
26
|
+
}
|
|
27
|
+
function getSeedWordCount(value) {
|
|
28
|
+
const normalized = normalizeKeyword(value);
|
|
29
|
+
if (!normalized)
|
|
30
|
+
return 0;
|
|
31
|
+
return normalized.split(/\s+/).filter(Boolean).length;
|
|
32
|
+
}
|
|
33
|
+
function isHighQualitySerpSeed(value) {
|
|
34
|
+
const normalized = normalizeKeyword(value);
|
|
35
|
+
if (!normalized)
|
|
36
|
+
return false;
|
|
37
|
+
const tokens = normalized.split(/\s+/).filter(Boolean);
|
|
38
|
+
if (tokens.length === 1) {
|
|
39
|
+
return tokens[0].length >= SERP_MIN_SHORT_KEYWORD_LENGTH;
|
|
40
|
+
}
|
|
41
|
+
if (tokens.some((token) => token.length < 3)) {
|
|
42
|
+
return false;
|
|
43
|
+
}
|
|
44
|
+
const shortTokenCount = tokens.filter((token) => token.length < SERP_MIN_SHORT_KEYWORD_LENGTH).length;
|
|
45
|
+
if (shortTokenCount > 0) {
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
function sortSeedCandidatesByWeightThenAlpha(a, b) {
|
|
51
|
+
if (a.sourceWeight !== b.sourceWeight) {
|
|
52
|
+
return b.sourceWeight - a.sourceWeight;
|
|
53
|
+
}
|
|
54
|
+
return a.keyword.localeCompare(b.keyword);
|
|
55
|
+
}
|
|
56
|
+
function dedupeByPermutation(seeds) {
|
|
57
|
+
const deduplicated = new Map();
|
|
58
|
+
for (const seed of seeds) {
|
|
59
|
+
const key = normalizeSerpSeedPermutationKey(seed.keyword);
|
|
60
|
+
if (!key)
|
|
61
|
+
continue;
|
|
62
|
+
const existing = deduplicated.get(key);
|
|
63
|
+
if (!existing) {
|
|
64
|
+
deduplicated.set(key, seed);
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
if (seed.sourceWeight > existing.sourceWeight) {
|
|
68
|
+
deduplicated.set(key, seed);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return [...deduplicated.values()];
|
|
72
|
+
}
|
|
73
|
+
export function buildSerpCampaignSeeds(pages, topKeywordsPerPage, maxSeeds) {
|
|
74
|
+
const shortBuckets = new Map();
|
|
75
|
+
const longBuckets = new Map();
|
|
76
|
+
const shortLimit = Math.max(1, topKeywordsPerPage);
|
|
77
|
+
const longCandidateLimit = Math.max(maxSeeds, shortLimit * 2);
|
|
78
|
+
const targetShortCount = Math.max(1, Math.min(maxSeeds, Math.ceil(maxSeeds * 0.45)));
|
|
79
|
+
for (const page of pages) {
|
|
80
|
+
if (!page.seoReport)
|
|
81
|
+
continue;
|
|
82
|
+
const shortSeeds = extractKeywordCampaignSeedsFromReport(page.seoReport, {
|
|
83
|
+
maxKeywords: shortLimit,
|
|
84
|
+
includeLongTail: false,
|
|
85
|
+
minKeywordLength: SERP_MIN_SHORT_KEYWORD_LENGTH,
|
|
86
|
+
sourcePage: page.url,
|
|
87
|
+
}).filter((seed) => isHighQualitySerpSeed(seed.keyword)).map((seed) => ({
|
|
88
|
+
keyword: seed.keyword,
|
|
89
|
+
sourcePage: seed.sourcePage,
|
|
90
|
+
sourceWeight: Math.max(1, Math.round(seed.weight)),
|
|
91
|
+
}));
|
|
92
|
+
const longCandidates = extractKeywordCampaignSeedsFromReport(page.seoReport, {
|
|
93
|
+
maxKeywords: longCandidateLimit,
|
|
94
|
+
minKeywordLength: SERP_MIN_LONG_TERM_LENGTH,
|
|
95
|
+
sourcePage: page.url,
|
|
96
|
+
}).filter((seed) => isHighQualitySerpSeed(seed.keyword));
|
|
97
|
+
const addSeed = (seed, target) => {
|
|
98
|
+
const key = normalizeSerpSeedPermutationKey(seed.keyword);
|
|
99
|
+
if (!key)
|
|
100
|
+
return;
|
|
101
|
+
const existing = target.get(key);
|
|
102
|
+
if (!existing) {
|
|
103
|
+
target.set(key, { ...seed });
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
existing.sourceWeight += seed.sourceWeight;
|
|
107
|
+
if (!existing.sourcePage && seed.sourcePage) {
|
|
108
|
+
existing.sourcePage = seed.sourcePage;
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
for (const item of shortSeeds.slice(0, shortLimit)) {
|
|
112
|
+
addSeed(item, shortBuckets);
|
|
113
|
+
}
|
|
114
|
+
for (const item of longCandidates) {
|
|
115
|
+
if (getSeedWordCount(item.keyword) <= 1)
|
|
116
|
+
continue;
|
|
117
|
+
addSeed({
|
|
118
|
+
keyword: item.keyword,
|
|
119
|
+
sourcePage: page.url,
|
|
120
|
+
sourceWeight: Math.max(1, Math.round(item.weight)),
|
|
121
|
+
}, longBuckets);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
const shortSeedList = dedupeByPermutation([...shortBuckets.values()].sort(sortSeedCandidatesByWeightThenAlpha));
|
|
125
|
+
const shortCount = Math.min(targetShortCount, shortSeedList.length);
|
|
126
|
+
const shortRemaining = targetShortCount - shortCount;
|
|
127
|
+
const longCount = Math.max(0, maxSeeds - shortCount);
|
|
128
|
+
const longBudget = Math.max(longCount + shortRemaining, 0);
|
|
129
|
+
const longSeedList = dedupeByPermutation([...longBuckets.values()].sort((a, b) => {
|
|
130
|
+
const aWords = getSeedWordCount(a.keyword);
|
|
131
|
+
const bWords = getSeedWordCount(b.keyword);
|
|
132
|
+
if (aWords !== bWords)
|
|
133
|
+
return aWords - bWords;
|
|
134
|
+
return sortSeedCandidatesByWeightThenAlpha(a, b);
|
|
135
|
+
}));
|
|
136
|
+
const actualLongCount = Math.min(longBudget, longSeedList.length);
|
|
137
|
+
const extraShortBudget = longBudget - actualLongCount;
|
|
138
|
+
const canFillOrdered = shortSeedList.length + actualLongCount >= maxSeeds;
|
|
139
|
+
const finalShortCount = canFillOrdered
|
|
140
|
+
? Math.min(shortCount + extraShortBudget, shortSeedList.length)
|
|
141
|
+
: shortCount;
|
|
142
|
+
const ordered = [...shortSeedList.slice(0, finalShortCount), ...longSeedList.slice(0, longBudget)]
|
|
143
|
+
.slice(0, maxSeeds)
|
|
144
|
+
.sort((a, b) => {
|
|
145
|
+
const aWords = getSeedWordCount(a.keyword);
|
|
146
|
+
const bWords = getSeedWordCount(b.keyword);
|
|
147
|
+
if (aWords !== bWords)
|
|
148
|
+
return aWords - bWords;
|
|
149
|
+
return b.sourceWeight - a.sourceWeight;
|
|
150
|
+
});
|
|
151
|
+
return {
|
|
152
|
+
short: shortSeedList.slice(0, finalShortCount),
|
|
153
|
+
longTail: longSeedList,
|
|
154
|
+
ordered,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
export async function runCrawlerSerpCampaign(targetUrl, pages, config) {
|
|
158
|
+
const seeds = buildSerpCampaignSeeds(pages, config.topKeywordsLimit, config.queriesLimit);
|
|
159
|
+
if (seeds.ordered.length === 0)
|
|
160
|
+
return undefined;
|
|
161
|
+
const inferredLocale = inferSearchLocaleFromUrl(targetUrl);
|
|
162
|
+
const hasExplicitLocale = Boolean(config.searchOptions.country?.trim()
|
|
163
|
+
|| config.searchOptions.gl?.trim()
|
|
164
|
+
|| config.searchOptions.hl?.trim());
|
|
165
|
+
const resolvedSearchParams = {
|
|
166
|
+
...config.searchOptions,
|
|
167
|
+
};
|
|
168
|
+
if (!hasExplicitLocale && inferredLocale) {
|
|
169
|
+
resolvedSearchParams.country = inferredLocale.country;
|
|
170
|
+
resolvedSearchParams.gl = inferredLocale.gl;
|
|
171
|
+
resolvedSearchParams.hl = inferredLocale.hl;
|
|
172
|
+
}
|
|
173
|
+
const discoveredKeywords = seeds.ordered.map((seed) => ({
|
|
174
|
+
keyword: seed.keyword,
|
|
175
|
+
source: 'discovered',
|
|
176
|
+
sourcePage: seed.sourcePage,
|
|
177
|
+
weight: seed.sourceWeight,
|
|
178
|
+
}));
|
|
179
|
+
const campaign = await analyzeKeywordCampaign({
|
|
180
|
+
targetUrl,
|
|
181
|
+
discoveredKeywords,
|
|
182
|
+
maxQueries: config.queriesLimit,
|
|
183
|
+
maxResultsPerQuery: config.resultsPerQuery,
|
|
184
|
+
minKeywordLength: 2,
|
|
185
|
+
searchDelayMs: config.searchDelayMs,
|
|
186
|
+
searchDelayJitterMs: config.searchDelayJitterMs,
|
|
187
|
+
searchCaptchaCooldownMs: config.searchCaptchaCooldownMs,
|
|
188
|
+
searchMaxConsecutiveBlocks: config.searchMaxConsecutiveBlocks,
|
|
189
|
+
searchRetryCount: config.searchRetryCount,
|
|
190
|
+
searchRetryDelayMs: config.searchRetryDelayMs,
|
|
191
|
+
searchConcurrency: config.searchConcurrency,
|
|
192
|
+
searchParams: resolvedSearchParams,
|
|
193
|
+
});
|
|
194
|
+
return {
|
|
195
|
+
campaign,
|
|
196
|
+
plan: seeds,
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
export function formatSerpSummaryRows(campaign) {
|
|
200
|
+
return campaign.results.slice(0, 12).map((item) => ({
|
|
201
|
+
keyword: item.keyword,
|
|
202
|
+
found: item.found,
|
|
203
|
+
blocked: item.blocked,
|
|
204
|
+
blockReason: item.blockReason,
|
|
205
|
+
position: item.bestPosition,
|
|
206
|
+
targetUrl: item.matchedUrl,
|
|
207
|
+
searchUrl: item.searchUrl,
|
|
208
|
+
}));
|
|
209
|
+
}
|
|
210
|
+
export function getSerpComparisonRows(campaign) {
|
|
211
|
+
return campaign.pageComparison.map((item) => ({
|
|
212
|
+
pageUrl: item.pageUrl,
|
|
213
|
+
tracked: item.tracked,
|
|
214
|
+
found: item.found,
|
|
215
|
+
appearanceRate: `${item.appearanceRate.toFixed(1)}%`,
|
|
216
|
+
avgPosition: item.avgPosition === null ? 'n/a' : String(item.avgPosition),
|
|
217
|
+
top3: item.top3,
|
|
218
|
+
top10: item.top10,
|
|
219
|
+
}));
|
|
220
|
+
}
|
|
221
|
+
export function getSerpCompetitorRows(competitors) {
|
|
222
|
+
return competitors.map((item) => ({
|
|
223
|
+
domain: item.domain,
|
|
224
|
+
queries: item.organicQueries,
|
|
225
|
+
matchedKeywords: item.matchedKeywords,
|
|
226
|
+
wins: item.totalOutperformedQueries,
|
|
227
|
+
avgGap: item.avgOutperformedGap === null ? 'n/a' : String(item.avgOutperformedGap),
|
|
228
|
+
bestPages: item.outperformedPages
|
|
229
|
+
.slice(0, 2)
|
|
230
|
+
.map((page) => `${page.pageUrl} (${page.outperformedQueries})`)
|
|
231
|
+
.join(' | ') || '—',
|
|
232
|
+
}));
|
|
233
|
+
}
|
|
234
|
+
export function formatKeywordList(keywords, maxItems) {
|
|
235
|
+
const shown = keywords
|
|
236
|
+
.slice(0, maxItems)
|
|
237
|
+
.map((keyword) => keyword.trim())
|
|
238
|
+
.filter(Boolean);
|
|
239
|
+
if (shown.length === 0)
|
|
240
|
+
return ' —';
|
|
241
|
+
return ` ${shown.join(' · ')}`;
|
|
242
|
+
}
|
|
243
|
+
export { parseSpiderSerpConfig };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const sitemapHandler: (ctx: import("../handler-types.js").ExecutionContext) => Promise<void>;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { withHandler, colors, } from '../output.js';
|
|
2
|
+
import { normalizeUrl } from './seo-serp.js';
|
|
3
|
+
export const sitemapHandler = withHandler({ loading: true }, async (ctx, out, extCtx) => {
|
|
4
|
+
let url = ctx.result.positional.url;
|
|
5
|
+
if (!url) {
|
|
6
|
+
const base = extCtx?.baseUrl?.();
|
|
7
|
+
if (base) {
|
|
8
|
+
url = base;
|
|
9
|
+
}
|
|
10
|
+
else {
|
|
11
|
+
out.error('Usage: sitemap <url> or set base URL first');
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
url = normalizeUrl(url);
|
|
16
|
+
const sitemapUrl = url.endsWith('.xml') ? url : new URL('/sitemap.xml', url).toString();
|
|
17
|
+
let response;
|
|
18
|
+
if (extCtx?.client) {
|
|
19
|
+
response = await extCtx.client.get(sitemapUrl);
|
|
20
|
+
}
|
|
21
|
+
else {
|
|
22
|
+
response = await fetch(sitemapUrl);
|
|
23
|
+
}
|
|
24
|
+
const text = await response.text();
|
|
25
|
+
if (response.status === 404) {
|
|
26
|
+
out.warn(`No sitemap found at ${sitemapUrl}`);
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
const urlMatches = text.match(/<loc>([^<]+)<\/loc>/g) || [];
|
|
30
|
+
const urls = urlMatches.map(m => m.replace(/<\/?loc>/g, ''));
|
|
31
|
+
const result = {
|
|
32
|
+
url: sitemapUrl,
|
|
33
|
+
totalUrls: urls.length,
|
|
34
|
+
sample: urls.slice(0, 10),
|
|
35
|
+
};
|
|
36
|
+
if (extCtx) {
|
|
37
|
+
out.response(result, { responseType: 'sitemap' });
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
out.title('Sitemap Analysis', '🗺️');
|
|
41
|
+
out.keyValue({
|
|
42
|
+
URL: sitemapUrl,
|
|
43
|
+
'Total URLs': urls.length,
|
|
44
|
+
});
|
|
45
|
+
out.blank();
|
|
46
|
+
if (urls.length > 0) {
|
|
47
|
+
out.subtitle('Sample URLs');
|
|
48
|
+
out.list(urls.slice(0, 10));
|
|
49
|
+
if (urls.length > 10) {
|
|
50
|
+
out.log(colors.gray(` ... and ${urls.length - 10} more`));
|
|
51
|
+
}
|
|
52
|
+
out.blank();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const spiderHandler: (ctx: import("../handler-types.js").ExecutionContext) => Promise<void>;
|