recker 1.0.79 → 1.0.80-next.cdb4529
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/browser/mini.d.ts +2 -2
- package/dist/browser/browser/recker-mini.d.ts +8 -8
- package/dist/browser/browser/recker.d.ts +11 -8
- package/dist/browser/browser/recker.js +54 -6
- package/dist/browser/core/client.d.ts +15 -10
- package/dist/browser/core/client.js +54 -38
- package/dist/browser/index.iife.min.js +129 -130
- package/dist/browser/index.min.js +129 -130
- package/dist/browser/index.mini.iife.js +5697 -636
- package/dist/browser/index.mini.iife.min.js +47 -48
- package/dist/browser/index.mini.min.js +47 -48
- package/dist/browser/index.mini.umd.js +5697 -636
- package/dist/browser/index.mini.umd.min.js +47 -48
- package/dist/browser/index.umd.min.js +129 -130
- package/dist/browser/mini.d.ts +2 -2
- package/dist/browser/plugins/proxy-rotator.d.ts +2 -2
- package/dist/browser/plugins/proxy-rotator.js +6 -28
- package/dist/browser/recker-mini.d.ts +8 -8
- package/dist/browser/recker.d.ts +11 -8
- package/dist/browser/recker.js +54 -6
- package/dist/browser/scrape/document.js +2 -2
- package/dist/browser/scrape/element.js +7 -1
- package/dist/browser/scrape/parser/nodes/html.js +1 -1
- package/dist/browser/scrape/spider.d.ts +52 -0
- package/dist/browser/scrape/spider.js +620 -38
- package/dist/browser/scrape/types.d.ts +2 -0
- package/dist/browser/search/google.d.ts +26 -1
- package/dist/browser/search/google.js +427 -45
- package/dist/browser/seo/analyzer.d.ts +1 -0
- package/dist/browser/seo/analyzer.js +144 -1
- package/dist/browser/seo/index.d.ts +1 -1
- package/dist/browser/seo/keyword-campaign-analyzer.d.ts +2 -0
- package/dist/browser/seo/keyword-campaign-analyzer.js +538 -0
- package/dist/browser/seo/keyword-campaign-seed-advanced.d.ts +17 -0
- package/dist/browser/seo/keyword-campaign-seed-advanced.js +269 -0
- package/dist/browser/seo/keyword-campaign-seed-core.d.ts +29 -0
- package/dist/browser/seo/keyword-campaign-seed-core.js +525 -0
- package/dist/browser/seo/keyword-campaign-shared.d.ts +165 -0
- package/dist/browser/seo/keyword-campaign-shared.js +59 -0
- package/dist/browser/seo/keyword-campaign.d.ts +4 -107
- package/dist/browser/seo/keyword-campaign.js +2 -380
- package/dist/browser/seo/keywords.js +5 -22
- package/dist/browser/seo/types.d.ts +19 -0
- package/dist/browser/transport/curl.d.ts +5 -1
- package/dist/browser/transport/curl.js +207 -50
- package/dist/browser/transport/undici.d.ts +4 -3
- package/dist/browser/transport/undici.js +123 -49
- package/dist/browser/types/index.d.ts +9 -3
- package/dist/browser/utils/binary-manager.js +26 -3
- package/dist/browser/utils/block-detector.d.ts +8 -0
- package/dist/browser/utils/block-detector.js +542 -7
- package/dist/cli/commands/hls-runner.js +5 -4
- package/dist/cli/commands/live-runner.js +5 -4
- package/dist/cli/commands/loadtest-runner.js +3 -2
- package/dist/cli/commands/search.d.ts +2 -0
- package/dist/cli/commands/search.js +105 -0
- package/dist/cli/commands/seo-runner.js +9 -7
- package/dist/cli/commands/seo.js +140 -1
- package/dist/cli/commands/serve.js +75 -131
- package/dist/cli/commands/server-runner.js +59 -82
- package/dist/cli/commands/spider-runner.d.ts +37 -1
- package/dist/cli/commands/spider-runner.js +134 -10
- package/dist/cli/commands/spider.d.ts +18 -1
- package/dist/cli/commands/spider.js +457 -27
- package/dist/cli/events/handlers/cli.js +30 -1
- package/dist/cli/events/handlers/tui.js +26 -0
- package/dist/cli/events/types.d.ts +27 -0
- package/dist/cli/handler.d.ts +2 -12
- package/dist/cli/handler.js +20 -15
- package/dist/cli/handlers/protocols.js +39 -12
- package/dist/cli/handlers/search.d.ts +2 -0
- package/dist/cli/handlers/search.js +171 -0
- package/dist/cli/handlers/seo-analyze.d.ts +1 -0
- package/dist/cli/handlers/seo-analyze.js +666 -0
- package/dist/cli/handlers/seo-robots.d.ts +1 -0
- package/dist/cli/handlers/seo-robots.js +76 -0
- package/dist/cli/handlers/seo-serp.d.ts +54 -0
- package/dist/cli/handlers/seo-serp.js +243 -0
- package/dist/cli/handlers/seo-sitemap.d.ts +1 -0
- package/dist/cli/handlers/seo-sitemap.js +55 -0
- package/dist/cli/handlers/seo-spider.d.ts +1 -0
- package/dist/cli/handlers/seo-spider.js +334 -0
- package/dist/cli/handlers/seo.d.ts +8 -4
- package/dist/cli/handlers/seo.js +294 -442
- package/dist/cli/handlers/spider.js +94 -17
- package/dist/cli/handlers/streaming.js +5 -1
- package/dist/cli/index.js +11 -2
- package/dist/cli/presets.d.ts +1 -1
- package/dist/cli/presets.js +15 -4
- package/dist/cli/tui/app.js +6 -1
- package/dist/cli/tui/components/rich-response.d.ts +72 -0
- package/dist/cli/tui/components/rich-response.js +117 -0
- package/dist/cli/tui/executor-commands/background.js +30 -24
- package/dist/cli/tui/executor-commands/testing.js +3 -2
- package/dist/cli/tui/hooks/useDomains.d.ts +17 -0
- package/dist/cli/tui/hooks/useHelp.js +15 -2
- package/dist/cli/tui/job-manager.d.ts +4 -4
- package/dist/cli/tui/job-manager.js +5 -1
- package/dist/cli/tui/spider-tui.d.ts +63 -0
- package/dist/cli/tui/spider-tui.js +120 -2
- package/dist/cli/types.d.ts +12 -0
- package/dist/cli/types.js +1 -0
- package/dist/cli/utils/option-helpers.d.ts +10 -0
- package/dist/cli/utils/option-helpers.js +63 -0
- package/dist/cli/utils/score-color.d.ts +11 -0
- package/dist/cli/utils/score-color.js +11 -0
- package/dist/cli/utils/serp-campaign.d.ts +53 -0
- package/dist/cli/utils/serp-campaign.js +53 -0
- package/dist/cli/utils/serp-config.d.ts +26 -0
- package/dist/cli/utils/serp-config.js +125 -0
- package/dist/core/client.d.ts +15 -10
- package/dist/core/client.js +54 -38
- package/dist/mcp/cli.js +35 -34
- package/dist/mcp/client.d.ts +2 -2
- package/dist/mcp/client.js +20 -2
- package/dist/mcp/contract.d.ts +1 -1
- package/dist/mcp/profiles.js +5 -1
- package/dist/mcp/prompts/index.js +8 -4
- package/dist/mcp/resources/index.js +46 -23
- package/dist/mcp/server.js +9 -6
- package/dist/mcp/tools/protocols.js +9 -2
- package/dist/mcp/tools/registry.js +13 -3
- package/dist/mcp/tools/seo.js +427 -2
- package/dist/mcp/types.d.ts +5 -1
- package/dist/plugins/proxy-rotator.d.ts +2 -2
- package/dist/plugins/proxy-rotator.js +6 -28
- package/dist/recker.d.ts +10 -7
- package/dist/recker.js +56 -6
- package/dist/scrape/document.js +2 -2
- package/dist/scrape/element.js +7 -1
- package/dist/scrape/parser/nodes/html.js +1 -1
- package/dist/scrape/spider.d.ts +52 -0
- package/dist/scrape/spider.js +620 -38
- package/dist/scrape/types.d.ts +2 -0
- package/dist/search/google.d.ts +26 -1
- package/dist/search/google.js +427 -45
- package/dist/search/index.d.ts +1 -1
- package/dist/seo/analyzer.d.ts +1 -0
- package/dist/seo/analyzer.js +144 -1
- package/dist/seo/index.d.ts +1 -1
- package/dist/seo/keyword-campaign-analyzer.d.ts +2 -0
- package/dist/seo/keyword-campaign-analyzer.js +538 -0
- package/dist/seo/keyword-campaign-seed-advanced.d.ts +17 -0
- package/dist/seo/keyword-campaign-seed-advanced.js +269 -0
- package/dist/seo/keyword-campaign-seed-core.d.ts +29 -0
- package/dist/seo/keyword-campaign-seed-core.js +525 -0
- package/dist/seo/keyword-campaign-shared.d.ts +165 -0
- package/dist/seo/keyword-campaign-shared.js +59 -0
- package/dist/seo/keyword-campaign.d.ts +4 -107
- package/dist/seo/keyword-campaign.js +2 -380
- package/dist/seo/keywords.js +5 -22
- package/dist/seo/types.d.ts +19 -0
- package/dist/template/index.d.ts +1 -1
- package/dist/template/types.d.ts +0 -2
- package/dist/testing/index.d.ts +1 -22
- package/dist/testing/index.js +1 -11
- package/dist/transport/curl.d.ts +5 -1
- package/dist/transport/curl.js +207 -50
- package/dist/transport/undici.d.ts +4 -3
- package/dist/transport/undici.js +123 -49
- package/dist/types/index.d.ts +9 -3
- package/dist/utils/binary-manager.js +26 -3
- package/dist/utils/block-detector.d.ts +8 -0
- package/dist/utils/block-detector.js +542 -7
- package/dist/version.js +1 -1
- package/package.json +8 -1
- package/dist/testing/mock-dns-server.d.ts +0 -69
- package/dist/testing/mock-dns-server.js +0 -269
- package/dist/testing/mock-ftp-server.d.ts +0 -89
- package/dist/testing/mock-ftp-server.js +0 -562
- package/dist/testing/mock-hls-server.d.ts +0 -80
- package/dist/testing/mock-hls-server.js +0 -381
- package/dist/testing/mock-http-server.d.ts +0 -124
- package/dist/testing/mock-http-server.js +0 -343
- package/dist/testing/mock-proxy-server.d.ts +0 -108
- package/dist/testing/mock-proxy-server.js +0 -615
- package/dist/testing/mock-sse-server.d.ts +0 -76
- package/dist/testing/mock-sse-server.js +0 -291
- package/dist/testing/mock-telnet-server.d.ts +0 -59
- package/dist/testing/mock-telnet-server.js +0 -274
- package/dist/testing/mock-udp-server.d.ts +0 -43
- package/dist/testing/mock-udp-server.js +0 -188
- package/dist/testing/mock-websocket-server.d.ts +0 -76
- package/dist/testing/mock-websocket-server.js +0 -334
- package/dist/testing/mock-whois-server.d.ts +0 -56
- package/dist/testing/mock-whois-server.js +0 -234
- package/dist/testing/proxy-certs.d.ts +0 -19
- package/dist/testing/proxy-certs.js +0 -208
package/dist/seo/analyzer.js
CHANGED
|
@@ -39,6 +39,7 @@ export class SeoAnalyzer {
|
|
|
39
39
|
const socialLinkDetails = this.analyzeSocialLinks(links, socialDomains);
|
|
40
40
|
const headings = this.analyzeHeadings();
|
|
41
41
|
const content = this.analyzeContent(headings);
|
|
42
|
+
const contentSections = this.extractContentSectionSignals(headings);
|
|
42
43
|
const linkAnalysis = this.buildLinkAnalysis(links);
|
|
43
44
|
const imageAnalysis = this.buildImageAnalysis(images);
|
|
44
45
|
const social = this.buildSocialAnalysis(og, twitter);
|
|
@@ -48,6 +49,16 @@ export class SeoAnalyzer {
|
|
|
48
49
|
const feeds = this.analyzeFeeds();
|
|
49
50
|
const conversion = this.analyzeConversionElements(links, visibleText);
|
|
50
51
|
const pageType = this.detectPageType(jsonLd);
|
|
52
|
+
const linkAnchorTexts = links
|
|
53
|
+
.map((link) => link.text?.trim())
|
|
54
|
+
.filter((text) => Boolean(text))
|
|
55
|
+
.filter((text) => text.length <= 140)
|
|
56
|
+
.filter((text) => !text.match(/^\s*$/))
|
|
57
|
+
.slice(0, 80);
|
|
58
|
+
const linkUrlSamples = links
|
|
59
|
+
.map((link) => extractReadablePathFromUrl(link.href))
|
|
60
|
+
.filter((sample) => sample.length > 0)
|
|
61
|
+
.slice(0, 80);
|
|
51
62
|
const context = this.buildRuleContext({
|
|
52
63
|
pageType,
|
|
53
64
|
meta,
|
|
@@ -124,12 +135,15 @@ export class SeoAnalyzer {
|
|
|
124
135
|
items: jsonLd,
|
|
125
136
|
},
|
|
126
137
|
headings: headings,
|
|
138
|
+
contentSections,
|
|
127
139
|
content,
|
|
128
140
|
keywords,
|
|
129
141
|
links: linkAnalysis,
|
|
130
142
|
images: imageAnalysis,
|
|
131
143
|
social,
|
|
132
144
|
technical,
|
|
145
|
+
linkAnchorTexts,
|
|
146
|
+
linkUrlSamples,
|
|
133
147
|
};
|
|
134
148
|
}
|
|
135
149
|
getMainBody() {
|
|
@@ -181,6 +195,97 @@ export class SeoAnalyzer {
|
|
|
181
195
|
return 'other';
|
|
182
196
|
}
|
|
183
197
|
}
|
|
198
|
+
extractContentSectionSignals(headings) {
|
|
199
|
+
const body = this.getMainBody();
|
|
200
|
+
if (!body)
|
|
201
|
+
return [];
|
|
202
|
+
const candidateTags = 'h1,h2,h3,h4,h5,h6,p,li,ol,ul,article,section,main,figure,figcaption,blockquote,table,th,td';
|
|
203
|
+
const nodes = body.querySelectorAll(candidateTags);
|
|
204
|
+
const normalizeSignalText = (value) => value
|
|
205
|
+
.replace(/\s+/g, ' ')
|
|
206
|
+
.replace(/[,\.;:!?()[\]{}"“”']/g, '')
|
|
207
|
+
.trim()
|
|
208
|
+
.toLowerCase();
|
|
209
|
+
const headingStack = [];
|
|
210
|
+
const seen = new Set();
|
|
211
|
+
const results = [];
|
|
212
|
+
const pushSectionSignal = (entry) => {
|
|
213
|
+
const fingerprint = `${entry.tagName}|${entry.headingPath.join(' > ')}|${entry.text.slice(0, 120)}`;
|
|
214
|
+
if (seen.has(fingerprint))
|
|
215
|
+
return;
|
|
216
|
+
seen.add(fingerprint);
|
|
217
|
+
results.push(entry);
|
|
218
|
+
};
|
|
219
|
+
for (const node of nodes) {
|
|
220
|
+
const tagName = node.tagName.toLowerCase();
|
|
221
|
+
if (/^h[1-6]$/.test(tagName)) {
|
|
222
|
+
const level = Number.parseInt(tagName.slice(1), 10);
|
|
223
|
+
const text = normalizeSignalText(node.text);
|
|
224
|
+
if (!text || text.length < 4) {
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
while (headingStack.length > 0 && headingStack[headingStack.length - 1].level >= level) {
|
|
228
|
+
headingStack.pop();
|
|
229
|
+
}
|
|
230
|
+
headingStack.push({ level, text });
|
|
231
|
+
const headingPath = headingStack.map((entry) => entry.text);
|
|
232
|
+
const source = {
|
|
233
|
+
headingPath,
|
|
234
|
+
source: 'heading',
|
|
235
|
+
tagName,
|
|
236
|
+
headingLevel: level,
|
|
237
|
+
text,
|
|
238
|
+
wordCount: text.split(/\s+/).length,
|
|
239
|
+
linkDensity: 0,
|
|
240
|
+
weight: 1.45,
|
|
241
|
+
};
|
|
242
|
+
pushSectionSignal(source);
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
if (tagName === 'ul' || tagName === 'ol') {
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
248
|
+
const text = normalizeSignalText(node.text);
|
|
249
|
+
if (!text || text.split(/\s+/).length < 4) {
|
|
250
|
+
continue;
|
|
251
|
+
}
|
|
252
|
+
const words = text.split(/\s+/);
|
|
253
|
+
const wordCount = words.length;
|
|
254
|
+
if (wordCount < 4)
|
|
255
|
+
continue;
|
|
256
|
+
const anchorCount = node.querySelectorAll('a').length;
|
|
257
|
+
const linkDensity = wordCount > 0
|
|
258
|
+
? Math.min(1, anchorCount / Math.max(1, wordCount))
|
|
259
|
+
: 0;
|
|
260
|
+
if (linkDensity > 0.85)
|
|
261
|
+
continue;
|
|
262
|
+
const headingPath = headingStack.length > 0
|
|
263
|
+
? headingStack.map((entry) => entry.text)
|
|
264
|
+
: headings.h1Count > 0
|
|
265
|
+
? ['conteúdo']
|
|
266
|
+
: ['home'];
|
|
267
|
+
const sourceLabel = tagName === 'li' ? 'list-item'
|
|
268
|
+
: tagName === 'figure' || tagName === 'figcaption' ? 'figure'
|
|
269
|
+
: (tagName === 'table' || tagName === 'th' || tagName === 'td') ? 'table'
|
|
270
|
+
: 'paragraph';
|
|
271
|
+
const headingLevel = headingStack.length > 0 ? headingStack[headingStack.length - 1].level : 1;
|
|
272
|
+
const baseWeight = sourceLabel === 'paragraph' ? 1.25 : 1.05;
|
|
273
|
+
const adjustedWeight = baseWeight * (1 - (linkDensity * 0.4));
|
|
274
|
+
pushSectionSignal({
|
|
275
|
+
headingPath,
|
|
276
|
+
source: sourceLabel,
|
|
277
|
+
tagName,
|
|
278
|
+
headingLevel,
|
|
279
|
+
text,
|
|
280
|
+
wordCount,
|
|
281
|
+
linkDensity,
|
|
282
|
+
weight: Math.max(0.5, adjustedWeight),
|
|
283
|
+
});
|
|
284
|
+
}
|
|
285
|
+
return results
|
|
286
|
+
.filter((entry) => entry.text.length >= 20)
|
|
287
|
+
.slice(0, 160);
|
|
288
|
+
}
|
|
184
289
|
getVisibleText() {
|
|
185
290
|
const body = this.getMainBody();
|
|
186
291
|
if (!body)
|
|
@@ -1221,7 +1326,7 @@ export class SeoAnalyzer {
|
|
|
1221
1326
|
const level = parseInt(tagName.substring(1), 10);
|
|
1222
1327
|
const text = el.text.trim();
|
|
1223
1328
|
counts[level] = (counts[level] || 0) + 1;
|
|
1224
|
-
structure.push({ level, text: text.slice(0,
|
|
1329
|
+
structure.push({ level, text: text.slice(0, 120), count: 1 });
|
|
1225
1330
|
if (level === 2) {
|
|
1226
1331
|
if (inSection) {
|
|
1227
1332
|
sectionWordCounts.push(currentSectionWordCount);
|
|
@@ -1261,8 +1366,24 @@ export class SeoAnalyzer {
|
|
|
1261
1366
|
else if (counts[1] > 1) {
|
|
1262
1367
|
issues.push('Multiple H1 tags');
|
|
1263
1368
|
}
|
|
1369
|
+
const tree = [];
|
|
1370
|
+
const stack = [];
|
|
1371
|
+
for (const heading of structure) {
|
|
1372
|
+
const node = { level: heading.level, text: heading.text, children: [] };
|
|
1373
|
+
while (stack.length > 0 && stack[stack.length - 1].level >= heading.level) {
|
|
1374
|
+
stack.pop();
|
|
1375
|
+
}
|
|
1376
|
+
if (stack.length === 0) {
|
|
1377
|
+
tree.push(node);
|
|
1378
|
+
}
|
|
1379
|
+
else {
|
|
1380
|
+
stack[stack.length - 1].node.children.push(node);
|
|
1381
|
+
}
|
|
1382
|
+
stack.push({ node, level: heading.level });
|
|
1383
|
+
}
|
|
1264
1384
|
return {
|
|
1265
1385
|
structure,
|
|
1386
|
+
tree,
|
|
1266
1387
|
h1Count: counts[1],
|
|
1267
1388
|
hasProperHierarchy,
|
|
1268
1389
|
issues,
|
|
@@ -1495,6 +1616,28 @@ export class SeoAnalyzer {
|
|
|
1495
1616
|
return this.rulesEngine.getCategories();
|
|
1496
1617
|
}
|
|
1497
1618
|
}
|
|
1619
|
+
function extractReadablePathFromUrl(value) {
|
|
1620
|
+
try {
|
|
1621
|
+
const parsed = new URL(value);
|
|
1622
|
+
const fragments = parsed.pathname
|
|
1623
|
+
.split('/')
|
|
1624
|
+
.filter((segment) => segment.length > 0)
|
|
1625
|
+
.slice(0, 3)
|
|
1626
|
+
.map((segment) => segment.replace(/[-_]/g, ' '))
|
|
1627
|
+
.join(' ')
|
|
1628
|
+
.replace(/\d+/g, ' ')
|
|
1629
|
+
.replace(/\s+/g, ' ')
|
|
1630
|
+
.trim();
|
|
1631
|
+
const host = parsed.hostname.replace(/^www\./, '').replace(/\./g, ' ');
|
|
1632
|
+
if (fragments.length > 0) {
|
|
1633
|
+
return `${host} ${fragments}`.trim();
|
|
1634
|
+
}
|
|
1635
|
+
return host;
|
|
1636
|
+
}
|
|
1637
|
+
catch {
|
|
1638
|
+
return '';
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1498
1641
|
export async function analyzeSeo(html, options = {}) {
|
|
1499
1642
|
const analyzer = await SeoAnalyzer.fromHtml(html, options);
|
|
1500
1643
|
return analyzer.analyze();
|
package/dist/seo/index.d.ts
CHANGED
|
@@ -6,7 +6,7 @@ export type { SeoSpiderOptions, SeoPageResult, SiteWideIssue, SeoSpiderResult, }
|
|
|
6
6
|
export { SeoRulesEngine, createRulesEngine, SEO_THRESHOLDS, ALL_SEO_RULES, } from './rules/index.js';
|
|
7
7
|
export { generateSeoFilename, resolveOutputPath, writeReport, formatReportForJson, } from './output.js';
|
|
8
8
|
export type { SeoOutputType, OutputOptions, WriteOptions } from './output.js';
|
|
9
|
-
export type { SeoReport, SeoCheckResult, SeoStatus, SeoPageType, SeoTiming, HeadingAnalysis, HeadingInfo, ContentMetrics, LinkAnalysis, ImageAnalysis, SocialMetaAnalysis, TechnicalSeo, SeoAnalyzerOptions, } from './types.js';
|
|
9
|
+
export type { SeoReport, SeoCheckResult, SeoStatus, SeoPageType, SeoTiming, HeadingAnalysis, HeadingInfo, HeadingNode, ContentMetrics, LinkAnalysis, ImageAnalysis, SocialMetaAnalysis, TechnicalSeo, SeoAnalyzerOptions, } from './types.js';
|
|
10
10
|
export type { SeoRule, RuleContext, RuleResult, RuleEvidence, RuleCategory, RuleSeverity, RulesEngineOptions, } from './rules/index.js';
|
|
11
11
|
export type { SeoAnalyzerFullOptions } from './analyzer.js';
|
|
12
12
|
export { parseRobotsTxt, validateRobotsTxt, isPathAllowed, fetchAndValidateRobotsTxt, } from './validators/robots.js';
|