recker 1.0.72 → 1.0.75-next.2e5a94f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -18
- package/dist/browser/core/client.d.ts +14 -8
- package/dist/browser/core/client.js +199 -17
- package/dist/browser/core/errors.d.ts +15 -1
- package/dist/browser/core/errors.js +140 -9
- package/dist/browser/core/request.d.ts +5 -0
- package/dist/browser/core/request.js +33 -2
- package/dist/browser/core-runtime/plugin-manifest.d.ts +24 -0
- package/dist/browser/core-runtime/plugin-manifest.js +159 -0
- package/dist/browser/core-runtime/request-context.d.ts +13 -0
- package/dist/browser/core-runtime/request-context.js +24 -0
- package/dist/browser/core-runtime/typed-events.d.ts +89 -0
- package/dist/browser/core-runtime/typed-events.js +34 -0
- package/dist/browser/index.iife.min.js +79 -79
- package/dist/browser/index.min.js +79 -79
- package/dist/browser/index.mini.iife.js +913 -97
- package/dist/browser/index.mini.iife.min.js +46 -46
- package/dist/browser/index.mini.min.js +46 -46
- package/dist/browser/index.mini.umd.js +913 -97
- package/dist/browser/index.mini.umd.min.js +46 -46
- package/dist/browser/index.umd.min.js +79 -79
- package/dist/browser/plugins/auth/aws-sigv4.d.ts +1 -0
- package/dist/browser/plugins/auth/aws-sigv4.js +19 -2
- package/dist/browser/plugins/retry.js +29 -1
- package/dist/browser/presets/aws.d.ts +1 -0
- package/dist/browser/presets/aws.js +62 -1
- package/dist/browser/runner/request-runner.d.ts +15 -5
- package/dist/browser/runner/request-runner.js +164 -30
- package/dist/browser/scrape/parser/nodes/html.d.ts +6 -0
- package/dist/browser/scrape/parser/nodes/html.js +70 -18
- package/dist/browser/scrape/parser/nodes/node.d.ts +1 -0
- package/dist/browser/scrape/parser/nodes/node.js +5 -0
- package/dist/browser/scrape/spider.d.ts +1 -0
- package/dist/browser/scrape/spider.js +39 -26
- package/dist/browser/seo/analyzer.d.ts +1 -1
- package/dist/browser/seo/analyzer.js +73 -42
- package/dist/browser/seo/index.d.ts +1 -1
- package/dist/browser/seo/rules/types.d.ts +2 -0
- package/dist/browser/seo/seo-spider.d.ts +2 -3
- package/dist/browser/seo/seo-spider.js +26 -202
- package/dist/browser/seo/types.d.ts +4 -0
- package/dist/browser/seo/validators/sitemap.js +9 -2
- package/dist/browser/transport/fetch.js +38 -5
- package/dist/browser/transport/undici.js +73 -11
- package/dist/browser/transport/worker.d.ts +0 -1
- package/dist/browser/transport/worker.js +1 -3
- package/dist/browser/types/index.d.ts +24 -0
- package/dist/cli/commands/mcp.js +5 -3
- package/dist/core/client.d.ts +14 -8
- package/dist/core/client.js +199 -17
- package/dist/core/errors.d.ts +15 -1
- package/dist/core/errors.js +140 -9
- package/dist/core/request.d.ts +5 -0
- package/dist/core/request.js +33 -2
- package/dist/core-runtime/plugin-manifest.d.ts +24 -0
- package/dist/core-runtime/plugin-manifest.js +159 -0
- package/dist/core-runtime/request-context.d.ts +13 -0
- package/dist/core-runtime/request-context.js +24 -0
- package/dist/core-runtime/typed-events.d.ts +89 -0
- package/dist/core-runtime/typed-events.js +34 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/mcp/cli.js +10 -8
- package/dist/mcp/profiles.d.ts +1 -1
- package/dist/mcp/profiles.js +31 -6
- package/dist/mcp/tools/categories.js +0 -1
- package/dist/mcp/tools/seo.js +320 -4
- package/dist/plugins/auth/aws-sigv4.d.ts +1 -0
- package/dist/plugins/auth/aws-sigv4.js +19 -2
- package/dist/plugins/retry.js +29 -1
- package/dist/presets/aws.d.ts +1 -0
- package/dist/presets/aws.js +62 -1
- package/dist/recker.d.ts +3 -0
- package/dist/recker.js +5 -0
- package/dist/runner/request-runner.d.ts +15 -5
- package/dist/runner/request-runner.js +164 -30
- package/dist/scrape/parser/nodes/html.d.ts +6 -0
- package/dist/scrape/parser/nodes/html.js +70 -18
- package/dist/scrape/parser/nodes/node.d.ts +1 -0
- package/dist/scrape/parser/nodes/node.js +5 -0
- package/dist/scrape/spider.d.ts +1 -0
- package/dist/scrape/spider.js +39 -26
- package/dist/search/google.d.ts +67 -0
- package/dist/search/google.js +480 -0
- package/dist/search/index.d.ts +3 -0
- package/dist/search/index.js +1 -0
- package/dist/seo/analyzer.d.ts +1 -1
- package/dist/seo/analyzer.js +73 -42
- package/dist/seo/index.d.ts +1 -1
- package/dist/seo/rules/types.d.ts +2 -0
- package/dist/seo/seo-spider.d.ts +2 -3
- package/dist/seo/seo-spider.js +26 -202
- package/dist/seo/types.d.ts +4 -0
- package/dist/seo/validators/sitemap.js +9 -2
- package/dist/transport/fetch.js +38 -5
- package/dist/transport/undici.js +73 -11
- package/dist/transport/worker.d.ts +0 -1
- package/dist/transport/worker.js +1 -3
- package/dist/types/index.d.ts +24 -0
- package/dist/version.js +1 -1
- package/package.json +9 -1
package/dist/mcp/cli.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { RekCommand as Command } from '../cli/router.js';
|
|
3
3
|
import { MCPServer } from './server.js';
|
|
4
|
-
import { listCategories, validateCategories, estimateCategoryTokens, } from './profiles.js';
|
|
4
|
+
import { listCategories, DEFAULT_CATEGORY, validateCategories, estimateCategoryTokens, } from './profiles.js';
|
|
5
5
|
import { LEGACY_TOOL_GROUPS } from './legacy-tool-groups.js';
|
|
6
6
|
const program = new Command('recker-mcp');
|
|
7
7
|
program
|
|
@@ -13,7 +13,7 @@ program
|
|
|
13
13
|
.option('--docs-path <path>', 'Path to documentation directory')
|
|
14
14
|
.option('--examples-path <path>', 'Path to examples directory')
|
|
15
15
|
.option('--src-path <path>', 'Path to source directory')
|
|
16
|
-
.option('-c, --category <categories>', 'Tool categories to enable (comma-separated): minimal, docs, network, dns, seo, security, scrape, video, ai, protocols, parsing, streaming, full')
|
|
16
|
+
.option('-c, --category <categories>', 'Tool categories to enable (comma-separated): minimal, docs, network, dns, seo, security, scrape, video, ai, protocols, parsing, streaming, template, full')
|
|
17
17
|
.option('--list-categories', 'List available categories and exit')
|
|
18
18
|
.option('--no-docs', 'Disable documentation tools (search, get, examples, schema, suggest)')
|
|
19
19
|
.option('--no-http', 'Disable HTTP request tool')
|
|
@@ -50,6 +50,7 @@ program
|
|
|
50
50
|
console.log('');
|
|
51
51
|
process.exit(0);
|
|
52
52
|
}
|
|
53
|
+
const useExplicitCategory = Boolean(opts.category);
|
|
53
54
|
if (opts.category) {
|
|
54
55
|
const categoryNames = opts.category.split(',').map((p) => p.trim());
|
|
55
56
|
const validation = validateCategories(categoryNames);
|
|
@@ -99,6 +100,7 @@ program
|
|
|
99
100
|
console.error(`Invalid transport mode: ${transport}. Use: stdio, http, or sse`);
|
|
100
101
|
process.exit(1);
|
|
101
102
|
}
|
|
103
|
+
const effectiveCategory = useExplicitCategory ? opts.category : (!opts.only && !opts.filter ? DEFAULT_CATEGORY : undefined);
|
|
102
104
|
const server = new MCPServer({
|
|
103
105
|
transport,
|
|
104
106
|
port,
|
|
@@ -106,8 +108,8 @@ program
|
|
|
106
108
|
docsPath: opts.docsPath,
|
|
107
109
|
examplesPath: opts.examplesPath,
|
|
108
110
|
srcPath: opts.srcPath,
|
|
109
|
-
category:
|
|
110
|
-
toolsFilter: !
|
|
111
|
+
category: effectiveCategory,
|
|
112
|
+
toolsFilter: !effectiveCategory && toolsFilter.length > 0 ? toolsFilter : undefined,
|
|
111
113
|
});
|
|
112
114
|
if (transport !== 'stdio') {
|
|
113
115
|
console.log('╔═══════════════════════════════════════════════════════════════════╗');
|
|
@@ -117,15 +119,15 @@ program
|
|
|
117
119
|
console.log(` Transport: ${transport}`);
|
|
118
120
|
console.log(` Port: ${port}`);
|
|
119
121
|
console.log(` Debug: ${opts.debug ? 'enabled' : 'disabled'}`);
|
|
120
|
-
if (
|
|
121
|
-
const tokens = estimateCategoryTokens(
|
|
122
|
-
console.log(` Category: ${
|
|
122
|
+
if (effectiveCategory) {
|
|
123
|
+
const tokens = estimateCategoryTokens(effectiveCategory);
|
|
124
|
+
console.log(` Category: ${effectiveCategory} (~${tokens} tokens)`);
|
|
123
125
|
}
|
|
124
126
|
else if (toolsFilter.length > 0) {
|
|
125
127
|
console.log(` Filters: ${toolsFilter.join(', ')}`);
|
|
126
128
|
}
|
|
127
129
|
else {
|
|
128
|
-
console.log(` Category:
|
|
130
|
+
console.log(` Category: minimal (default)`);
|
|
129
131
|
}
|
|
130
132
|
console.log('');
|
|
131
133
|
console.log(' Available tools:');
|
package/dist/mcp/profiles.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type CategoryName = 'minimal' | 'docs' | 'network' | 'dns' | 'seo' | 'security' | 'scrape' | 'video' | 'ai' | 'protocols' | 'parsing' | 'streaming' | 'full';
|
|
1
|
+
export type CategoryName = 'minimal' | 'docs' | 'network' | 'dns' | 'seo' | 'security' | 'scrape' | 'video' | 'ai' | 'protocols' | 'parsing' | 'streaming' | 'template' | 'full';
|
|
2
2
|
export interface Category {
|
|
3
3
|
name: CategoryName;
|
|
4
4
|
description: string;
|
package/dist/mcp/profiles.js
CHANGED
|
@@ -118,11 +118,10 @@ export const categories = {
|
|
|
118
118
|
},
|
|
119
119
|
ai: {
|
|
120
120
|
name: 'ai',
|
|
121
|
-
description: 'Multi-provider AI chat
|
|
121
|
+
description: 'Multi-provider AI chat and comparison',
|
|
122
122
|
icon: '🤖',
|
|
123
123
|
tools: [
|
|
124
124
|
'rek_ai_chat',
|
|
125
|
-
'rek_ai_embed',
|
|
126
125
|
'rek_ai_providers',
|
|
127
126
|
'rek_ai_tokens',
|
|
128
127
|
'rek_ai_compare',
|
|
@@ -173,14 +172,39 @@ export const categories = {
|
|
|
173
172
|
],
|
|
174
173
|
estimatedTokens: 900,
|
|
175
174
|
},
|
|
175
|
+
template: {
|
|
176
|
+
name: 'template',
|
|
177
|
+
description: 'Template rendering, validation, parsing, and helper metadata',
|
|
178
|
+
icon: '📝',
|
|
179
|
+
tools: [
|
|
180
|
+
'rek_template_render',
|
|
181
|
+
'rek_template_validate',
|
|
182
|
+
'rek_template_parse',
|
|
183
|
+
'rek_template_variables',
|
|
184
|
+
'rek_template_check',
|
|
185
|
+
'rek_template_helpers',
|
|
186
|
+
],
|
|
187
|
+
estimatedTokens: 1800,
|
|
188
|
+
},
|
|
176
189
|
full: {
|
|
177
190
|
name: 'full',
|
|
178
191
|
description: 'All available tools (high context cost)',
|
|
179
192
|
icon: '🌟',
|
|
180
193
|
tools: ['*'],
|
|
181
|
-
estimatedTokens:
|
|
194
|
+
estimatedTokens: 0,
|
|
182
195
|
},
|
|
183
196
|
};
|
|
197
|
+
function getAllConcreteProfileTools() {
|
|
198
|
+
const toolNames = new Set();
|
|
199
|
+
for (const category of Object.values(categories)) {
|
|
200
|
+
for (const tool of category.tools) {
|
|
201
|
+
if (tool !== '*') {
|
|
202
|
+
toolNames.add(tool);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return toolNames;
|
|
207
|
+
}
|
|
184
208
|
export const DEFAULT_CATEGORY = 'minimal';
|
|
185
209
|
export const profiles = categories;
|
|
186
210
|
export const DEFAULT_PROFILE = DEFAULT_CATEGORY;
|
|
@@ -226,7 +250,7 @@ export function estimateCategoryTokens(categoryNames) {
|
|
|
226
250
|
if (!category)
|
|
227
251
|
continue;
|
|
228
252
|
if (category.tools.includes('*')) {
|
|
229
|
-
return
|
|
253
|
+
return getAllConcreteProfileTools().size * 300;
|
|
230
254
|
}
|
|
231
255
|
for (const tool of category.tools) {
|
|
232
256
|
if (!seenTools.has(tool)) {
|
|
@@ -239,12 +263,13 @@ export function estimateCategoryTokens(categoryNames) {
|
|
|
239
263
|
}
|
|
240
264
|
export const estimateProfileTokens = estimateCategoryTokens;
|
|
241
265
|
export function listCategories() {
|
|
266
|
+
const toolCount = getAllConcreteProfileTools().size;
|
|
242
267
|
return Object.values(categories).map((p) => ({
|
|
243
268
|
name: p.name,
|
|
244
269
|
description: p.description,
|
|
245
270
|
icon: p.icon,
|
|
246
|
-
toolCount: p.tools.includes('*') ?
|
|
247
|
-
estimatedTokens: p.estimatedTokens,
|
|
271
|
+
toolCount: p.tools.includes('*') ? toolCount : p.tools.length,
|
|
272
|
+
estimatedTokens: p.estimatedTokens || (p.tools.includes('*') ? toolCount * 300 : p.tools.length * 300),
|
|
248
273
|
}));
|
|
249
274
|
}
|
|
250
275
|
export const listProfiles = listCategories;
|
package/dist/mcp/tools/seo.js
CHANGED
|
@@ -1,6 +1,114 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { createHash } from 'node:crypto';
|
|
1
5
|
import { createClient } from '../../core/client.js';
|
|
2
6
|
import { analyzeSeo } from '../../seo/analyzer.js';
|
|
3
7
|
import { seoSpider } from '../../seo/seo-spider.js';
|
|
8
|
+
const DEFAULT_SEO_CACHE_TTL_SEC = 6 * 60 * 60;
|
|
9
|
+
const SEO_REPORT_DIR = join(tmpdir(), 'recker', 'seo');
|
|
10
|
+
function normalizeStringArray(value) {
|
|
11
|
+
if (!Array.isArray(value))
|
|
12
|
+
return undefined;
|
|
13
|
+
const items = value.map(item => String(item).trim()).filter(Boolean);
|
|
14
|
+
if (items.length === 0)
|
|
15
|
+
return undefined;
|
|
16
|
+
return items;
|
|
17
|
+
}
|
|
18
|
+
function safeHost(url) {
|
|
19
|
+
try {
|
|
20
|
+
const parsed = new URL(url);
|
|
21
|
+
const host = parsed.hostname || parsed.host;
|
|
22
|
+
return host.replace(/[^a-z0-9.-]/gi, '_') || 'unknown';
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return url.replace(/[^a-z0-9.-]/gi, '_').slice(0, 64) || 'unknown';
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
function hashArgs(value) {
|
|
29
|
+
return createHash('sha1').update(JSON.stringify(value)).digest('hex').slice(0, 12);
|
|
30
|
+
}
|
|
31
|
+
function resolveReportPersistence(args, options) {
|
|
32
|
+
const output = typeof args.output === 'string' ? args.output : undefined;
|
|
33
|
+
const outputDir = typeof args.outputDir === 'string' ? args.outputDir : undefined;
|
|
34
|
+
const persist = args.persist === true;
|
|
35
|
+
const persistenceEnabled = Boolean(output || outputDir || persist);
|
|
36
|
+
const cacheEnabled = persistenceEnabled ? (typeof args.cache === 'boolean' ? args.cache : true) : false;
|
|
37
|
+
const forceRefresh = args.forceRefresh === true;
|
|
38
|
+
const cacheTtlInput = typeof args.cacheTtlSec === 'number'
|
|
39
|
+
? args.cacheTtlSec
|
|
40
|
+
: Number.isFinite(Number(args.cacheTtlSec))
|
|
41
|
+
? Number(args.cacheTtlSec)
|
|
42
|
+
: undefined;
|
|
43
|
+
const cacheTtlSec = cacheEnabled
|
|
44
|
+
? Math.max(0, cacheTtlInput ?? DEFAULT_SEO_CACHE_TTL_SEC)
|
|
45
|
+
: 0;
|
|
46
|
+
if (!persistenceEnabled) {
|
|
47
|
+
return {
|
|
48
|
+
enabled: false,
|
|
49
|
+
reportPath: null,
|
|
50
|
+
cacheEnabled: false,
|
|
51
|
+
cacheTtlSec: 0,
|
|
52
|
+
forceRefresh,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
const host = safeHost(options.url);
|
|
56
|
+
const hash = hashArgs(options.cacheKey);
|
|
57
|
+
const fileName = `${options.toolSlug}--${host}--${hash}.json`;
|
|
58
|
+
const reportPath = output
|
|
59
|
+
? output
|
|
60
|
+
: outputDir
|
|
61
|
+
? join(outputDir, fileName)
|
|
62
|
+
: join(SEO_REPORT_DIR, fileName);
|
|
63
|
+
return {
|
|
64
|
+
enabled: true,
|
|
65
|
+
reportPath,
|
|
66
|
+
cacheEnabled,
|
|
67
|
+
cacheTtlSec,
|
|
68
|
+
forceRefresh,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
async function readCachedReport(path, ttlSec) {
|
|
72
|
+
if (!path || ttlSec <= 0)
|
|
73
|
+
return null;
|
|
74
|
+
try {
|
|
75
|
+
const stats = await fs.stat(path);
|
|
76
|
+
const ageSec = (Date.now() - stats.mtimeMs) / 1000;
|
|
77
|
+
if (ageSec > ttlSec) {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
const text = await fs.readFile(path, 'utf8');
|
|
81
|
+
const report = JSON.parse(text);
|
|
82
|
+
if (report && typeof report === 'object') {
|
|
83
|
+
delete report.reportMeta;
|
|
84
|
+
delete report.note;
|
|
85
|
+
}
|
|
86
|
+
return {
|
|
87
|
+
report,
|
|
88
|
+
ageSec,
|
|
89
|
+
bytes: stats.size,
|
|
90
|
+
savedAt: new Date(stats.mtimeMs).toISOString(),
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
async function writeReport(path, payload) {
|
|
98
|
+
await fs.mkdir(dirname(path), { recursive: true });
|
|
99
|
+
await fs.writeFile(path, payload, 'utf8');
|
|
100
|
+
return { bytes: Buffer.byteLength(payload), savedAt: new Date().toISOString() };
|
|
101
|
+
}
|
|
102
|
+
function buildReportMeta(params) {
|
|
103
|
+
return {
|
|
104
|
+
reportPath: params.reportPath ?? null,
|
|
105
|
+
cacheHit: params.cacheHit ?? false,
|
|
106
|
+
source: params.source,
|
|
107
|
+
...(params.cacheAgeSec !== undefined ? { cacheAgeSec: params.cacheAgeSec } : {}),
|
|
108
|
+
...(params.savedAt ? { savedAt: params.savedAt } : {}),
|
|
109
|
+
...(params.reportBytes !== undefined ? { reportBytes: params.reportBytes } : {}),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
4
112
|
function formatCheck(check) {
|
|
5
113
|
const icon = check.status === 'pass' ? '✓' : check.status === 'fail' ? '✗' : '⚠';
|
|
6
114
|
let line = `${icon} [${check.status.toUpperCase()}] ${check.name}: ${check.message}`;
|
|
@@ -75,14 +183,46 @@ function generateQuickWins(report) {
|
|
|
75
183
|
}
|
|
76
184
|
async function seoAnalyze(args) {
|
|
77
185
|
const url = String(args.url || '');
|
|
78
|
-
const categories = args.categories;
|
|
186
|
+
const categories = normalizeStringArray(args.categories);
|
|
79
187
|
if (!url) {
|
|
80
188
|
return {
|
|
81
189
|
content: [{ type: 'text', text: 'Error: url is required' }],
|
|
82
190
|
isError: true,
|
|
83
191
|
};
|
|
84
192
|
}
|
|
193
|
+
const persistence = resolveReportPersistence(args, {
|
|
194
|
+
toolSlug: 'seo',
|
|
195
|
+
url,
|
|
196
|
+
cacheKey: {
|
|
197
|
+
url,
|
|
198
|
+
categories: categories ? [...categories].sort() : [],
|
|
199
|
+
},
|
|
200
|
+
});
|
|
85
201
|
try {
|
|
202
|
+
if (persistence.enabled && persistence.cacheEnabled && !persistence.forceRefresh && persistence.reportPath) {
|
|
203
|
+
const cached = await readCachedReport(persistence.reportPath, persistence.cacheTtlSec);
|
|
204
|
+
if (cached) {
|
|
205
|
+
const reportMeta = buildReportMeta({
|
|
206
|
+
reportPath: persistence.reportPath,
|
|
207
|
+
cacheHit: true,
|
|
208
|
+
cacheAgeSec: cached.ageSec,
|
|
209
|
+
savedAt: cached.savedAt,
|
|
210
|
+
reportBytes: cached.bytes,
|
|
211
|
+
source: 'cached',
|
|
212
|
+
});
|
|
213
|
+
const output = {
|
|
214
|
+
...cached.report,
|
|
215
|
+
reportMeta,
|
|
216
|
+
note: `Cache hit: report loaded from ${persistence.reportPath}`,
|
|
217
|
+
};
|
|
218
|
+
return {
|
|
219
|
+
content: [{
|
|
220
|
+
type: 'text',
|
|
221
|
+
text: JSON.stringify(output, null, 2),
|
|
222
|
+
}],
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
}
|
|
86
226
|
const client = createClient({ timeout: 30000 });
|
|
87
227
|
const response = await client.get(url);
|
|
88
228
|
const html = await response.text();
|
|
@@ -135,10 +275,56 @@ async function seoAnalyze(args) {
|
|
|
135
275
|
images: report.images,
|
|
136
276
|
technical: report.technical,
|
|
137
277
|
};
|
|
278
|
+
const reportPayload = JSON.stringify(output, null, 2);
|
|
279
|
+
let reportMeta = buildReportMeta({
|
|
280
|
+
reportPath: persistence.reportPath,
|
|
281
|
+
cacheHit: false,
|
|
282
|
+
source: 'fresh',
|
|
283
|
+
});
|
|
284
|
+
let note = persistence.enabled
|
|
285
|
+
? `Report generated (not yet saved).`
|
|
286
|
+
: 'Report generated (not persisted). Set persist=true to save and enable caching.';
|
|
287
|
+
if (persistence.enabled && persistence.reportPath) {
|
|
288
|
+
try {
|
|
289
|
+
const saved = await writeReport(persistence.reportPath, reportPayload);
|
|
290
|
+
reportMeta = buildReportMeta({
|
|
291
|
+
reportPath: persistence.reportPath,
|
|
292
|
+
cacheHit: false,
|
|
293
|
+
savedAt: saved.savedAt,
|
|
294
|
+
reportBytes: saved.bytes,
|
|
295
|
+
source: 'fresh',
|
|
296
|
+
});
|
|
297
|
+
note = `Report saved to ${persistence.reportPath}`;
|
|
298
|
+
}
|
|
299
|
+
catch (error) {
|
|
300
|
+
const message = `Failed to save report to ${persistence.reportPath}: ${error.message}`;
|
|
301
|
+
const failedOutput = {
|
|
302
|
+
...output,
|
|
303
|
+
reportMeta,
|
|
304
|
+
note: message,
|
|
305
|
+
};
|
|
306
|
+
return {
|
|
307
|
+
content: [{
|
|
308
|
+
type: 'text',
|
|
309
|
+
text: JSON.stringify(failedOutput, null, 2),
|
|
310
|
+
}],
|
|
311
|
+
isError: true,
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
else {
|
|
316
|
+
reportMeta = buildReportMeta({
|
|
317
|
+
reportPath: null,
|
|
318
|
+
cacheHit: false,
|
|
319
|
+
reportBytes: Buffer.byteLength(reportPayload),
|
|
320
|
+
savedAt: new Date().toISOString(),
|
|
321
|
+
source: 'fresh',
|
|
322
|
+
});
|
|
323
|
+
}
|
|
138
324
|
return {
|
|
139
325
|
content: [{
|
|
140
326
|
type: 'text',
|
|
141
|
-
text: JSON.stringify(output, null, 2),
|
|
327
|
+
text: JSON.stringify({ ...output, reportMeta, note }, null, 2),
|
|
142
328
|
}],
|
|
143
329
|
};
|
|
144
330
|
}
|
|
@@ -165,6 +351,41 @@ async function seoSpiderCrawl(args) {
|
|
|
165
351
|
};
|
|
166
352
|
}
|
|
167
353
|
try {
|
|
354
|
+
const persistence = resolveReportPersistence(args, {
|
|
355
|
+
toolSlug: 'seo-spider',
|
|
356
|
+
url,
|
|
357
|
+
cacheKey: {
|
|
358
|
+
url,
|
|
359
|
+
maxPages,
|
|
360
|
+
maxDepth,
|
|
361
|
+
concurrency,
|
|
362
|
+
transport,
|
|
363
|
+
},
|
|
364
|
+
});
|
|
365
|
+
if (persistence.enabled && persistence.cacheEnabled && !persistence.forceRefresh && persistence.reportPath) {
|
|
366
|
+
const cached = await readCachedReport(persistence.reportPath, persistence.cacheTtlSec);
|
|
367
|
+
if (cached) {
|
|
368
|
+
const reportMeta = buildReportMeta({
|
|
369
|
+
reportPath: persistence.reportPath,
|
|
370
|
+
cacheHit: true,
|
|
371
|
+
cacheAgeSec: cached.ageSec,
|
|
372
|
+
savedAt: cached.savedAt,
|
|
373
|
+
reportBytes: cached.bytes,
|
|
374
|
+
source: 'cached',
|
|
375
|
+
});
|
|
376
|
+
const output = {
|
|
377
|
+
...cached.report,
|
|
378
|
+
reportMeta,
|
|
379
|
+
note: `Cache hit: report loaded from ${persistence.reportPath}`,
|
|
380
|
+
};
|
|
381
|
+
return {
|
|
382
|
+
content: [{
|
|
383
|
+
type: 'text',
|
|
384
|
+
text: JSON.stringify(output, null, 2),
|
|
385
|
+
}],
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
}
|
|
168
389
|
const result = await seoSpider(url, {
|
|
169
390
|
seo: true,
|
|
170
391
|
maxPages,
|
|
@@ -226,10 +447,56 @@ async function seoSpiderCrawl(args) {
|
|
|
226
447
|
if (recommendations.length > 0) {
|
|
227
448
|
output.recommendations = recommendations;
|
|
228
449
|
}
|
|
450
|
+
const reportPayload = JSON.stringify(output, null, 2);
|
|
451
|
+
let reportMeta = buildReportMeta({
|
|
452
|
+
reportPath: persistence.reportPath,
|
|
453
|
+
cacheHit: false,
|
|
454
|
+
source: 'fresh',
|
|
455
|
+
});
|
|
456
|
+
let note = persistence.enabled
|
|
457
|
+
? 'Report generated (not yet saved).'
|
|
458
|
+
: 'Report generated (not persisted). Set persist=true to save and enable caching.';
|
|
459
|
+
if (persistence.enabled && persistence.reportPath) {
|
|
460
|
+
try {
|
|
461
|
+
const saved = await writeReport(persistence.reportPath, reportPayload);
|
|
462
|
+
reportMeta = buildReportMeta({
|
|
463
|
+
reportPath: persistence.reportPath,
|
|
464
|
+
cacheHit: false,
|
|
465
|
+
savedAt: saved.savedAt,
|
|
466
|
+
reportBytes: saved.bytes,
|
|
467
|
+
source: 'fresh',
|
|
468
|
+
});
|
|
469
|
+
note = `Report saved to ${persistence.reportPath}`;
|
|
470
|
+
}
|
|
471
|
+
catch (error) {
|
|
472
|
+
const message = `Failed to save report to ${persistence.reportPath}: ${error.message}`;
|
|
473
|
+
const failedOutput = {
|
|
474
|
+
...output,
|
|
475
|
+
reportMeta,
|
|
476
|
+
note: message,
|
|
477
|
+
};
|
|
478
|
+
return {
|
|
479
|
+
content: [{
|
|
480
|
+
type: 'text',
|
|
481
|
+
text: JSON.stringify(failedOutput, null, 2),
|
|
482
|
+
}],
|
|
483
|
+
isError: true,
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
else {
|
|
488
|
+
reportMeta = buildReportMeta({
|
|
489
|
+
reportPath: null,
|
|
490
|
+
cacheHit: false,
|
|
491
|
+
reportBytes: Buffer.byteLength(reportPayload),
|
|
492
|
+
savedAt: new Date().toISOString(),
|
|
493
|
+
source: 'fresh',
|
|
494
|
+
});
|
|
495
|
+
}
|
|
229
496
|
return {
|
|
230
497
|
content: [{
|
|
231
498
|
type: 'text',
|
|
232
|
-
text: JSON.stringify(output, null, 2),
|
|
499
|
+
text: JSON.stringify({ ...output, reportMeta, note }, null, 2),
|
|
233
500
|
}],
|
|
234
501
|
};
|
|
235
502
|
}
|
|
@@ -330,6 +597,7 @@ Returns:
|
|
|
330
597
|
- Warnings and recommendations
|
|
331
598
|
- OpenGraph/social meta analysis
|
|
332
599
|
- Request timing breakdown
|
|
600
|
+
- Optional report persistence and caching for AI agent workflows
|
|
333
601
|
|
|
334
602
|
Perfect for analyzing your localhost dev server or any public URL. Categories include: meta, content, links, images, technical, security, performance, mobile, accessibility, schema, structural, i18n, PWA, social, e-commerce, local SEO, Core Web Vitals, readability, crawlability, internal linking, and best practices.`,
|
|
335
603
|
inputSchema: {
|
|
@@ -344,6 +612,30 @@ Perfect for analyzing your localhost dev server or any public URL. Categories in
|
|
|
344
612
|
items: { type: 'string' },
|
|
345
613
|
description: 'Filter by specific categories (e.g., ["meta", "security", "performance"]). Leave empty for all.',
|
|
346
614
|
},
|
|
615
|
+
output: {
|
|
616
|
+
type: 'string',
|
|
617
|
+
description: 'Save report to explicit file path',
|
|
618
|
+
},
|
|
619
|
+
outputDir: {
|
|
620
|
+
type: 'string',
|
|
621
|
+
description: 'Save report to directory with auto-generated filename',
|
|
622
|
+
},
|
|
623
|
+
persist: {
|
|
624
|
+
type: 'boolean',
|
|
625
|
+
description: 'Save report to default temp path (os.tmpdir()/recker/seo)',
|
|
626
|
+
},
|
|
627
|
+
cache: {
|
|
628
|
+
type: 'boolean',
|
|
629
|
+
description: 'Reuse existing saved report if present and within TTL (default: true when persistence enabled)',
|
|
630
|
+
},
|
|
631
|
+
cacheTtlSec: {
|
|
632
|
+
type: 'number',
|
|
633
|
+
description: `Cache TTL in seconds (default: ${DEFAULT_SEO_CACHE_TTL_SEC})`,
|
|
634
|
+
},
|
|
635
|
+
forceRefresh: {
|
|
636
|
+
type: 'boolean',
|
|
637
|
+
description: 'Bypass cache and recompute even if a saved report exists',
|
|
638
|
+
},
|
|
347
639
|
},
|
|
348
640
|
required: ['url'],
|
|
349
641
|
},
|
|
@@ -357,7 +649,7 @@ Detects site-wide issues:
|
|
|
357
649
|
- Orphan pages (no internal links pointing to them)
|
|
358
650
|
- Pages with low SEO scores
|
|
359
651
|
|
|
360
|
-
Returns per-page scores and prioritized recommendations for improving overall site SEO. Great for auditing a full site before launch or finding issues across your dev environment.`,
|
|
652
|
+
Returns per-page scores and prioritized recommendations for improving overall site SEO. Great for auditing a full site before launch or finding issues across your dev environment. Supports optional report persistence and caching for AI agent workflows.`,
|
|
361
653
|
inputSchema: {
|
|
362
654
|
type: 'object',
|
|
363
655
|
properties: {
|
|
@@ -386,6 +678,30 @@ Returns per-page scores and prioritized recommendations for improving overall si
|
|
|
386
678
|
description: 'HTTP transport: auto (try undici, fallback to curl on WAF block), undici (fast), curl (curl-impersonate for protected sites)',
|
|
387
679
|
default: 'auto',
|
|
388
680
|
},
|
|
681
|
+
output: {
|
|
682
|
+
type: 'string',
|
|
683
|
+
description: 'Save report to explicit file path',
|
|
684
|
+
},
|
|
685
|
+
outputDir: {
|
|
686
|
+
type: 'string',
|
|
687
|
+
description: 'Save report to directory with auto-generated filename',
|
|
688
|
+
},
|
|
689
|
+
persist: {
|
|
690
|
+
type: 'boolean',
|
|
691
|
+
description: 'Save report to default temp path (os.tmpdir()/recker/seo)',
|
|
692
|
+
},
|
|
693
|
+
cache: {
|
|
694
|
+
type: 'boolean',
|
|
695
|
+
description: 'Reuse existing saved report if present and within TTL (default: true when persistence enabled)',
|
|
696
|
+
},
|
|
697
|
+
cacheTtlSec: {
|
|
698
|
+
type: 'number',
|
|
699
|
+
description: `Cache TTL in seconds (default: ${DEFAULT_SEO_CACHE_TTL_SEC})`,
|
|
700
|
+
},
|
|
701
|
+
forceRefresh: {
|
|
702
|
+
type: 'boolean',
|
|
703
|
+
description: 'Bypass cache and recompute even if a saved report exists',
|
|
704
|
+
},
|
|
389
705
|
},
|
|
390
706
|
required: ['url'],
|
|
391
707
|
},
|
|
@@ -8,3 +8,4 @@ export interface AWSSignatureV4Options {
|
|
|
8
8
|
}
|
|
9
9
|
export declare function awsSignatureV4(options: AWSSignatureV4Options): Middleware;
|
|
10
10
|
export declare function awsSignatureV4Plugin(options: AWSSignatureV4Options): Plugin;
|
|
11
|
+
export declare function clearAwsSigV4PluginCache(): void;
|
|
@@ -81,8 +81,25 @@ export function awsSignatureV4(options) {
|
|
|
81
81
|
return next(newReq);
|
|
82
82
|
};
|
|
83
83
|
}
|
|
84
|
+
const awsSigV4PluginCache = new Map();
|
|
84
85
|
export function awsSignatureV4Plugin(options) {
|
|
85
|
-
|
|
86
|
-
|
|
86
|
+
const cacheKey = [
|
|
87
|
+
options.accessKeyId,
|
|
88
|
+
options.secretAccessKey,
|
|
89
|
+
options.region,
|
|
90
|
+
options.service,
|
|
91
|
+
options.sessionToken || '',
|
|
92
|
+
].join('|');
|
|
93
|
+
const cached = awsSigV4PluginCache.get(cacheKey);
|
|
94
|
+
if (cached)
|
|
95
|
+
return cached;
|
|
96
|
+
const safeOptions = { ...options };
|
|
97
|
+
const plugin = (client) => {
|
|
98
|
+
client.use(awsSignatureV4(safeOptions));
|
|
87
99
|
};
|
|
100
|
+
awsSigV4PluginCache.set(cacheKey, plugin);
|
|
101
|
+
return plugin;
|
|
102
|
+
}
|
|
103
|
+
export function clearAwsSigV4PluginCache() {
|
|
104
|
+
awsSigV4PluginCache.clear();
|
|
88
105
|
}
|
package/dist/plugins/retry.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { HttpError, NetworkError, TimeoutError } from '../core/errors.js';
|
|
1
|
+
import { HttpError, NetworkError, TimeoutError, classifyTransportError } from '../core/errors.js';
|
|
2
|
+
import { getRequestContext } from '../core-runtime/request-context.js';
|
|
2
3
|
function calculateDelay(attempt, baseDelay, maxDelay, strategy, useJitter) {
|
|
3
4
|
let calculatedDelay;
|
|
4
5
|
switch (strategy) {
|
|
@@ -54,6 +55,13 @@ export function retryPlugin(options = {}) {
|
|
|
54
55
|
if (error instanceof HttpError) {
|
|
55
56
|
return statusCodes.includes(error.status);
|
|
56
57
|
}
|
|
58
|
+
const classification = classifyTransportError(error);
|
|
59
|
+
if (classification && classification.canRetry === false) {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
if (classification) {
|
|
63
|
+
return classification.canRetry;
|
|
64
|
+
}
|
|
57
65
|
if (error && typeof error === 'object' && 'code' in error) {
|
|
58
66
|
const code = error.code;
|
|
59
67
|
return code === 'ECONNRESET' || code === 'ETIMEDOUT' || code === 'ENOTFOUND';
|
|
@@ -62,6 +70,24 @@ export function retryPlugin(options = {}) {
|
|
|
62
70
|
};
|
|
63
71
|
const shouldRetry = options.shouldRetry || defaultShouldRetry;
|
|
64
72
|
return (client) => {
|
|
73
|
+
const emitRequestRetry = (error, attempt, delayMs, req) => {
|
|
74
|
+
const eventBus = client.runtimeEventBus;
|
|
75
|
+
if (!eventBus?.emit) {
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
const context = getRequestContext(req);
|
|
79
|
+
if (!context) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const classification = classifyTransportError(error);
|
|
83
|
+
eventBus.emit('request:retry', {
|
|
84
|
+
context,
|
|
85
|
+
req,
|
|
86
|
+
attempt,
|
|
87
|
+
delayMs,
|
|
88
|
+
reason: classification?.reason || error?.message || 'retry requested'
|
|
89
|
+
});
|
|
90
|
+
};
|
|
65
91
|
const middleware = async (req, next) => {
|
|
66
92
|
let attempt = 0;
|
|
67
93
|
while (true) {
|
|
@@ -80,6 +106,7 @@ export function retryPlugin(options = {}) {
|
|
|
80
106
|
delayMs = calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
|
|
81
107
|
}
|
|
82
108
|
const err = new HttpError(res, req);
|
|
109
|
+
emitRequestRetry(err, attempt, delayMs, req);
|
|
83
110
|
if (onRetry) {
|
|
84
111
|
onRetry(attempt, err, delayMs);
|
|
85
112
|
}
|
|
@@ -96,6 +123,7 @@ export function retryPlugin(options = {}) {
|
|
|
96
123
|
catch (error) {
|
|
97
124
|
if (attempt < maxAttempts && shouldRetry(error)) {
|
|
98
125
|
const delayMs = calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
|
|
126
|
+
emitRequestRetry(error, attempt, delayMs, req);
|
|
99
127
|
if (onRetry) {
|
|
100
128
|
onRetry(attempt, error, delayMs);
|
|
101
129
|
}
|
package/dist/presets/aws.d.ts
CHANGED
|
@@ -10,6 +10,7 @@ export interface AWSPresetOptions {
|
|
|
10
10
|
export type AWSService = 's3' | 'dynamodb' | 'lambda' | 'sqs' | 'sns' | 'ses' | 'secretsmanager' | 'ssm' | 'sts' | 'iam' | 'ec2' | 'ecs' | 'eks' | 'cloudwatch' | 'logs' | 'events' | 'kinesis' | 'firehose' | 'apigateway' | 'execute-api' | 'cognito-idp' | 'cognito-identity' | 'kms' | 'athena' | 'glue' | 'stepfunctions' | 'states' | 'bedrock' | 'bedrock-runtime';
|
|
11
11
|
export declare function aws(options: AWSPresetOptions): ClientOptions;
|
|
12
12
|
export declare function awsS3(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
|
|
13
|
+
export declare function s3(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
|
|
13
14
|
export declare function awsDynamoDB(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
|
|
14
15
|
export declare function awsLambda(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
|
|
15
16
|
export declare function awsSQS(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
|