recker 1.0.9 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/handler.d.ts.map +1 -1
- package/dist/cli/handler.js +17 -3
- package/dist/cli/index.js +81 -2
- package/dist/cli/tui/shell.d.ts +14 -0
- package/dist/cli/tui/shell.d.ts.map +1 -1
- package/dist/cli/tui/shell.js +709 -19
- package/package.json +4 -7
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../src/cli/handler.ts"],"names":[],"mappings":"AAwBA,UAAU,cAAc;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,GAAG,CAAC;CACpB;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,cAAc,
|
|
1
|
+
{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../src/cli/handler.ts"],"names":[],"mappings":"AAwBA,UAAU,cAAc;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,YAAY,CAAC,EAAE,GAAG,CAAC;CACpB;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,cAAc,iBAqH1D"}
|
package/dist/cli/handler.js
CHANGED
|
@@ -39,9 +39,23 @@ export async function handleRequest(options) {
|
|
|
39
39
|
}
|
|
40
40
|
let requestBody = undefined;
|
|
41
41
|
if (options.body) {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
options.headers['Content-Type']
|
|
42
|
+
if (typeof options.body === 'string') {
|
|
43
|
+
requestBody = options.body;
|
|
44
|
+
if (!options.headers['Content-Type'] && !options.headers['content-type']) {
|
|
45
|
+
try {
|
|
46
|
+
JSON.parse(options.body);
|
|
47
|
+
options.headers['Content-Type'] = 'application/json';
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
options.headers['Content-Type'] = 'text/plain';
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
requestBody = JSON.stringify(options.body);
|
|
56
|
+
if (!options.headers['Content-Type'] && !options.headers['content-type']) {
|
|
57
|
+
options.headers['Content-Type'] = 'application/json';
|
|
58
|
+
}
|
|
45
59
|
}
|
|
46
60
|
}
|
|
47
61
|
const response = await client.request(options.url, {
|
package/dist/cli/index.js
CHANGED
|
@@ -1,6 +1,68 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { program } from 'commander';
|
|
3
|
+
import { promises as fs } from 'node:fs';
|
|
4
|
+
import { join } from 'node:path';
|
|
3
5
|
import pc from '../utils/colors.js';
|
|
6
|
+
async function readStdin() {
|
|
7
|
+
if (process.stdin.isTTY) {
|
|
8
|
+
return null;
|
|
9
|
+
}
|
|
10
|
+
return new Promise((resolve) => {
|
|
11
|
+
let data = '';
|
|
12
|
+
const timeout = setTimeout(() => {
|
|
13
|
+
resolve(null);
|
|
14
|
+
}, 100);
|
|
15
|
+
process.stdin.setEncoding('utf8');
|
|
16
|
+
process.stdin.on('data', (chunk) => {
|
|
17
|
+
clearTimeout(timeout);
|
|
18
|
+
data += chunk;
|
|
19
|
+
});
|
|
20
|
+
process.stdin.on('end', () => {
|
|
21
|
+
clearTimeout(timeout);
|
|
22
|
+
resolve(data.trim() || null);
|
|
23
|
+
});
|
|
24
|
+
process.stdin.on('error', () => {
|
|
25
|
+
clearTimeout(timeout);
|
|
26
|
+
resolve(null);
|
|
27
|
+
});
|
|
28
|
+
process.stdin.resume();
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
async function loadEnvFile(filePath) {
|
|
32
|
+
const envPath = typeof filePath === 'string' ? filePath : join(process.cwd(), '.env');
|
|
33
|
+
const envVars = {};
|
|
34
|
+
try {
|
|
35
|
+
const content = await fs.readFile(envPath, 'utf-8');
|
|
36
|
+
const lines = content.split('\n');
|
|
37
|
+
for (const line of lines) {
|
|
38
|
+
const trimmed = line.trim();
|
|
39
|
+
if (!trimmed || trimmed.startsWith('#'))
|
|
40
|
+
continue;
|
|
41
|
+
const match = trimmed.match(/^([^=]+)=(.*)$/);
|
|
42
|
+
if (match) {
|
|
43
|
+
const [, key, value] = match;
|
|
44
|
+
const cleanKey = key.trim();
|
|
45
|
+
let cleanValue = value.trim();
|
|
46
|
+
if ((cleanValue.startsWith('"') && cleanValue.endsWith('"')) ||
|
|
47
|
+
(cleanValue.startsWith("'") && cleanValue.endsWith("'"))) {
|
|
48
|
+
cleanValue = cleanValue.slice(1, -1);
|
|
49
|
+
}
|
|
50
|
+
envVars[cleanKey] = cleanValue;
|
|
51
|
+
process.env[cleanKey] = cleanValue;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
console.log(pc.gray(`Loaded ${Object.keys(envVars).length} variables from ${envPath}`));
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
if (error.code === 'ENOENT') {
|
|
58
|
+
console.log(pc.yellow(`Warning: No .env file found at ${envPath}`));
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
console.log(pc.red(`Error loading .env: ${error.message}`));
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return envVars;
|
|
65
|
+
}
|
|
4
66
|
async function main() {
|
|
5
67
|
const { handleRequest } = await import('./handler.js');
|
|
6
68
|
const { resolvePreset } = await import('./presets.js');
|
|
@@ -51,7 +113,7 @@ async function main() {
|
|
|
51
113
|
}
|
|
52
114
|
if (!url) {
|
|
53
115
|
url = arg;
|
|
54
|
-
if (!hasPreset && !url.startsWith('http') && !url.startsWith('ws') && !url.startsWith('udp')) {
|
|
116
|
+
if (!hasPreset && !url.startsWith('http://') && !url.startsWith('https://') && !url.startsWith('ws://') && !url.startsWith('wss://') && !url.startsWith('udp://')) {
|
|
55
117
|
url = `https://${url}`;
|
|
56
118
|
}
|
|
57
119
|
}
|
|
@@ -66,6 +128,7 @@ async function main() {
|
|
|
66
128
|
.argument('[args...]', 'URL, Method, Headers (Key:Value), Data (key=value)')
|
|
67
129
|
.option('-v, --verbose', 'Show full request/response details')
|
|
68
130
|
.option('-j, --json', 'Force JSON content-type')
|
|
131
|
+
.option('-e, --env [path]', 'Load .env file from current directory or specified path')
|
|
69
132
|
.addHelpText('after', `
|
|
70
133
|
${pc.bold(pc.yellow('Examples:'))}
|
|
71
134
|
${pc.green('$ rek httpbin.org/json')}
|
|
@@ -81,6 +144,10 @@ ${pc.bold(pc.yellow('Available Presets:'))}
|
|
|
81
144
|
program.help();
|
|
82
145
|
return;
|
|
83
146
|
}
|
|
147
|
+
if (options.env !== undefined) {
|
|
148
|
+
await loadEnvFile(options.env);
|
|
149
|
+
}
|
|
150
|
+
const stdinData = await readStdin();
|
|
84
151
|
let argsToParse = args;
|
|
85
152
|
let presetConfig = undefined;
|
|
86
153
|
if (args[0].startsWith('@')) {
|
|
@@ -116,11 +183,23 @@ ${pc.bold(pc.yellow('Available Presets:'))}
|
|
|
116
183
|
return;
|
|
117
184
|
}
|
|
118
185
|
try {
|
|
186
|
+
let body = undefined;
|
|
187
|
+
if (stdinData) {
|
|
188
|
+
try {
|
|
189
|
+
body = JSON.parse(stdinData);
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
body = stdinData;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
else if (Object.keys(data).length > 0) {
|
|
196
|
+
body = data;
|
|
197
|
+
}
|
|
119
198
|
await handleRequest({
|
|
120
199
|
method,
|
|
121
200
|
url,
|
|
122
201
|
headers,
|
|
123
|
-
body
|
|
202
|
+
body,
|
|
124
203
|
verbose: options.verbose,
|
|
125
204
|
presetConfig
|
|
126
205
|
});
|
package/dist/cli/tui/shell.d.ts
CHANGED
|
@@ -5,6 +5,8 @@ export declare class RekShell {
|
|
|
5
5
|
private baseUrl;
|
|
6
6
|
private lastResponse;
|
|
7
7
|
private variables;
|
|
8
|
+
private envVars;
|
|
9
|
+
private envLoaded;
|
|
8
10
|
private initialized;
|
|
9
11
|
private currentDoc;
|
|
10
12
|
private currentDocUrl;
|
|
@@ -23,6 +25,8 @@ export declare class RekShell {
|
|
|
23
25
|
private parseLine;
|
|
24
26
|
private setBaseUrl;
|
|
25
27
|
private setVariable;
|
|
28
|
+
private showVars;
|
|
29
|
+
loadEnvFile(filePath?: string): Promise<void>;
|
|
26
30
|
private resolveVariables;
|
|
27
31
|
private resolveUrl;
|
|
28
32
|
private executeRequest;
|
|
@@ -38,6 +42,16 @@ export declare class RekShell {
|
|
|
38
42
|
private runSelectHtml;
|
|
39
43
|
private runSelectLinks;
|
|
40
44
|
private runSelectImages;
|
|
45
|
+
private runSelectScripts;
|
|
46
|
+
private runSelectCSS;
|
|
47
|
+
private runSelectSourcemaps;
|
|
48
|
+
private runUnmap;
|
|
49
|
+
private runUnmapView;
|
|
50
|
+
private runUnmapSave;
|
|
51
|
+
private runBeautify;
|
|
52
|
+
private beautifyJS;
|
|
53
|
+
private beautifyCSS;
|
|
54
|
+
private runBeautifySave;
|
|
41
55
|
private runSelectTable;
|
|
42
56
|
private printHelp;
|
|
43
57
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shell.d.ts","sourceRoot":"","sources":["../../../src/cli/tui/shell.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"shell.d.ts","sourceRoot":"","sources":["../../../src/cli/tui/shell.ts"],"names":[],"mappings":"AAmCA,qBAAa,QAAQ;IACnB,OAAO,CAAC,EAAE,CAAsB;IAChC,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,SAAS,CAA2B;IAC5C,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,aAAa,CAAc;;YAWrB,iBAAiB;IAe/B,OAAO,CAAC,SAAS;IAMjB,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,aAAa;IAiBrB,OAAO,CAAC,SAAS;IAaJ,KAAK;IA8BlB,OAAO,CAAC,MAAM;YAKA,aAAa;YAmKb,kBAAkB;YAkBlB,SAAS;YAkBT,WAAW;IA0DzB,OAAO,CAAC,SAAS;IAMjB,OAAO,CAAC,UAAU;IAMlB,OAAO,CAAC,WAAW;IASnB,OAAO,CAAC,QAAQ;IAoCV,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM;IA6CnC,OAAO,CAAC,gBAAgB;IAqBxB,OAAO,CAAC,UAAU;YAeJ,cAAc;YAyEd,QAAQ;YA8GR,MAAM;YA2DN,MAAM;YA2EN,OAAO;YA+DP,OAAO;YA0CP,QAAQ;YAoER,SAAS;YAsCT,aAAa;YA8Bb,aAAa;YA+Bb,aAAa;YA6Bb,cAAc;YAkCd,eAAe;YA+Ef,gBAAgB;YAmEhB,YAAY;YAiEZ,mBAAmB;YAsFnB,QAAQ;YA0FR,YAAY;YAoCZ,YAAY;YA6CZ,WAAW;IA6CzB,OAAO,CAAC,UAAU;IA4GlB,OAAO,CAAC,WAAW;YAgFL,eAAe;YAkBf,cAAc;IA8C5B,OAAO,CAAC,SAAS;CAkElB"}
|
package/dist/cli/tui/shell.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import readline from 'node:readline';
|
|
2
2
|
import { promises as dns } from 'node:dns';
|
|
3
|
+
import { promises as fs } from 'node:fs';
|
|
4
|
+
import { join } from 'node:path';
|
|
3
5
|
import { requireOptional } from '../../utils/optional-require.js';
|
|
4
6
|
import { createClient } from '../../core/client.js';
|
|
5
7
|
import { startInteractiveWebSocket } from './websocket.js';
|
|
@@ -28,6 +30,8 @@ export class RekShell {
|
|
|
28
30
|
baseUrl = '';
|
|
29
31
|
lastResponse = null;
|
|
30
32
|
variables = {};
|
|
33
|
+
envVars = {};
|
|
34
|
+
envLoaded = false;
|
|
31
35
|
initialized = false;
|
|
32
36
|
currentDoc = null;
|
|
33
37
|
currentDocUrl = '';
|
|
@@ -81,8 +85,8 @@ export class RekShell {
|
|
|
81
85
|
'get', 'post', 'put', 'delete', 'patch', 'head', 'options',
|
|
82
86
|
'ws', 'udp', 'load', 'chat', 'ai',
|
|
83
87
|
'whois', 'tls', 'ssl', 'dns', 'rdap', 'ping',
|
|
84
|
-
'scrap', '$', '$text', '$attr', '$html', '$links', '$images', '$table',
|
|
85
|
-
'help', 'clear', 'exit', 'set', 'url', 'vars'
|
|
88
|
+
'scrap', '$', '$text', '$attr', '$html', '$links', '$images', '$scripts', '$css', '$sourcemaps', '$unmap', '$unmap:view', '$unmap:save', '$beautify', '$beautify:save', '$table',
|
|
89
|
+
'help', 'clear', 'exit', 'set', 'url', 'vars', 'env'
|
|
86
90
|
];
|
|
87
91
|
const hits = commands.filter((c) => c.startsWith(line));
|
|
88
92
|
return [hits.length ? hits : commands, line];
|
|
@@ -137,7 +141,10 @@ export class RekShell {
|
|
|
137
141
|
this.setVariable(parts.slice(1));
|
|
138
142
|
return;
|
|
139
143
|
case 'vars':
|
|
140
|
-
|
|
144
|
+
this.showVars();
|
|
145
|
+
return;
|
|
146
|
+
case 'env':
|
|
147
|
+
await this.loadEnvFile(parts[1]);
|
|
141
148
|
return;
|
|
142
149
|
case 'load':
|
|
143
150
|
await this.runLoadTest(parts.slice(1));
|
|
@@ -181,7 +188,31 @@ export class RekShell {
|
|
|
181
188
|
await this.runSelectLinks(parts[1]);
|
|
182
189
|
return;
|
|
183
190
|
case '$images':
|
|
184
|
-
await this.runSelectImages(parts
|
|
191
|
+
await this.runSelectImages(parts.slice(1).join(' ') || undefined);
|
|
192
|
+
return;
|
|
193
|
+
case '$scripts':
|
|
194
|
+
await this.runSelectScripts();
|
|
195
|
+
return;
|
|
196
|
+
case '$css':
|
|
197
|
+
await this.runSelectCSS();
|
|
198
|
+
return;
|
|
199
|
+
case '$sourcemaps':
|
|
200
|
+
await this.runSelectSourcemaps();
|
|
201
|
+
return;
|
|
202
|
+
case '$unmap':
|
|
203
|
+
await this.runUnmap(parts.slice(1).join(' '));
|
|
204
|
+
return;
|
|
205
|
+
case '$unmap:view':
|
|
206
|
+
await this.runUnmapView(parts[1] || '');
|
|
207
|
+
return;
|
|
208
|
+
case '$unmap:save':
|
|
209
|
+
await this.runUnmapSave(parts[1] || '');
|
|
210
|
+
return;
|
|
211
|
+
case '$beautify':
|
|
212
|
+
await this.runBeautify(parts.slice(1).join(' '));
|
|
213
|
+
return;
|
|
214
|
+
case '$beautify:save':
|
|
215
|
+
await this.runBeautifySave(parts[1] || '');
|
|
185
216
|
return;
|
|
186
217
|
case '$table':
|
|
187
218
|
await this.runSelectTable(parts.slice(1).join(' '));
|
|
@@ -332,6 +363,72 @@ export class RekShell {
|
|
|
332
363
|
this.variables[key] = val;
|
|
333
364
|
console.log(colors.gray(`Variable $${key} set.`));
|
|
334
365
|
}
|
|
366
|
+
showVars() {
|
|
367
|
+
const hasVars = Object.keys(this.variables).length > 0;
|
|
368
|
+
const hasEnvVars = Object.keys(this.envVars).length > 0;
|
|
369
|
+
if (!hasVars && !hasEnvVars) {
|
|
370
|
+
console.log(colors.gray('No variables set.'));
|
|
371
|
+
console.log(colors.gray('Use "set key=value" to set variables or "env" to load .env file.'));
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
if (hasVars) {
|
|
375
|
+
console.log(colors.bold(colors.yellow('\nSession Variables:')));
|
|
376
|
+
for (const [key, value] of Object.entries(this.variables)) {
|
|
377
|
+
console.log(` ${colors.cyan('$' + key)} = ${colors.green(String(value))}`);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
if (hasEnvVars) {
|
|
381
|
+
console.log(colors.bold(colors.yellow('\nEnvironment Variables (.env):')));
|
|
382
|
+
for (const [key, value] of Object.entries(this.envVars)) {
|
|
383
|
+
const displayValue = key.toLowerCase().includes('key') ||
|
|
384
|
+
key.toLowerCase().includes('secret') ||
|
|
385
|
+
key.toLowerCase().includes('password') ||
|
|
386
|
+
key.toLowerCase().includes('token')
|
|
387
|
+
? colors.gray('***' + value.slice(-4))
|
|
388
|
+
: colors.green(value);
|
|
389
|
+
console.log(` ${colors.cyan('$' + key)} = ${displayValue}`);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
console.log('');
|
|
393
|
+
}
|
|
394
|
+
async loadEnvFile(filePath) {
|
|
395
|
+
const envPath = filePath || join(process.cwd(), '.env');
|
|
396
|
+
try {
|
|
397
|
+
const content = await fs.readFile(envPath, 'utf-8');
|
|
398
|
+
const lines = content.split('\n');
|
|
399
|
+
let count = 0;
|
|
400
|
+
for (const line of lines) {
|
|
401
|
+
const trimmed = line.trim();
|
|
402
|
+
if (!trimmed || trimmed.startsWith('#'))
|
|
403
|
+
continue;
|
|
404
|
+
const match = trimmed.match(/^([^=]+)=(.*)$/);
|
|
405
|
+
if (match) {
|
|
406
|
+
const [, key, value] = match;
|
|
407
|
+
const cleanKey = key.trim();
|
|
408
|
+
let cleanValue = value.trim();
|
|
409
|
+
if ((cleanValue.startsWith('"') && cleanValue.endsWith('"')) ||
|
|
410
|
+
(cleanValue.startsWith("'") && cleanValue.endsWith("'"))) {
|
|
411
|
+
cleanValue = cleanValue.slice(1, -1);
|
|
412
|
+
}
|
|
413
|
+
this.envVars[cleanKey] = cleanValue;
|
|
414
|
+
process.env[cleanKey] = cleanValue;
|
|
415
|
+
count++;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
this.envLoaded = true;
|
|
419
|
+
console.log(colors.green(`✓ Loaded ${count} variables from ${colors.cyan(envPath)}`));
|
|
420
|
+
console.log(colors.gray('Use "vars" to list all variables.'));
|
|
421
|
+
}
|
|
422
|
+
catch (error) {
|
|
423
|
+
if (error.code === 'ENOENT') {
|
|
424
|
+
console.log(colors.yellow(`No .env file found at ${envPath}`));
|
|
425
|
+
console.log(colors.gray('Create a .env file with KEY=value pairs to use this feature.'));
|
|
426
|
+
}
|
|
427
|
+
else {
|
|
428
|
+
console.log(colors.red(`Error loading .env: ${error.message}`));
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
}
|
|
335
432
|
resolveVariables(value) {
|
|
336
433
|
if (value.startsWith('$')) {
|
|
337
434
|
const key = value.slice(1);
|
|
@@ -346,7 +443,7 @@ export class RekShell {
|
|
|
346
443
|
}
|
|
347
444
|
return String(current);
|
|
348
445
|
}
|
|
349
|
-
return this.variables[key] || value;
|
|
446
|
+
return this.variables[key] || this.envVars[key] || process.env[key] || value;
|
|
350
447
|
}
|
|
351
448
|
return value;
|
|
352
449
|
}
|
|
@@ -726,15 +823,43 @@ export class RekShell {
|
|
|
726
823
|
const response = await this.client.get(url);
|
|
727
824
|
const html = await response.text();
|
|
728
825
|
const duration = Math.round(performance.now() - startTime);
|
|
729
|
-
this.currentDoc = await ScrapeDocument.create(html);
|
|
826
|
+
this.currentDoc = await ScrapeDocument.create(html, { baseUrl: url });
|
|
730
827
|
this.currentDocUrl = url;
|
|
731
828
|
const elementCount = this.currentDoc.select('*').length;
|
|
732
829
|
const title = this.currentDoc.selectFirst('title').text() || 'No title';
|
|
830
|
+
const meta = this.currentDoc.meta();
|
|
831
|
+
const og = this.currentDoc.openGraph();
|
|
733
832
|
console.log(colors.green(`✔ Loaded`) + colors.gray(` (${duration}ms)`));
|
|
734
833
|
console.log(` ${colors.cyan('Title')}: ${title}`);
|
|
735
834
|
console.log(` ${colors.cyan('Elements')}: ${elementCount}`);
|
|
736
835
|
console.log(` ${colors.cyan('Size')}: ${(html.length / 1024).toFixed(1)}kb`);
|
|
737
|
-
|
|
836
|
+
if (meta.description) {
|
|
837
|
+
const desc = meta.description.length > 100 ? meta.description.slice(0, 100) + '...' : meta.description;
|
|
838
|
+
console.log(` ${colors.cyan('Description')}: ${desc}`);
|
|
839
|
+
}
|
|
840
|
+
const hasOg = og.title || og.description || og.image || og.siteName;
|
|
841
|
+
if (hasOg) {
|
|
842
|
+
console.log(colors.bold('\n OpenGraph:'));
|
|
843
|
+
if (og.siteName)
|
|
844
|
+
console.log(` ${colors.magenta('Site')}: ${og.siteName}`);
|
|
845
|
+
if (og.title && og.title !== title)
|
|
846
|
+
console.log(` ${colors.magenta('Title')}: ${og.title}`);
|
|
847
|
+
if (og.type)
|
|
848
|
+
console.log(` ${colors.magenta('Type')}: ${og.type}`);
|
|
849
|
+
if (og.description) {
|
|
850
|
+
const ogDesc = og.description.length > 80 ? og.description.slice(0, 80) + '...' : og.description;
|
|
851
|
+
console.log(` ${colors.magenta('Description')}: ${ogDesc}`);
|
|
852
|
+
}
|
|
853
|
+
if (og.image) {
|
|
854
|
+
const images = Array.isArray(og.image) ? og.image : [og.image];
|
|
855
|
+
console.log(` ${colors.magenta('Image')}: ${images[0]}`);
|
|
856
|
+
if (images.length > 1)
|
|
857
|
+
console.log(colors.gray(` (+${images.length - 1} more)`));
|
|
858
|
+
}
|
|
859
|
+
if (og.url && og.url !== url)
|
|
860
|
+
console.log(` ${colors.magenta('URL')}: ${og.url}`);
|
|
861
|
+
}
|
|
862
|
+
console.log(colors.gray('\n Use $ <selector> to query, $text, $attr, $links, $images, $scripts, $css, $sourcemaps, $table'));
|
|
738
863
|
}
|
|
739
864
|
catch (error) {
|
|
740
865
|
console.error(colors.red(`Scrape failed: ${error.message}`));
|
|
@@ -898,30 +1023,586 @@ export class RekShell {
|
|
|
898
1023
|
return;
|
|
899
1024
|
}
|
|
900
1025
|
try {
|
|
901
|
-
const
|
|
902
|
-
const elements = this.currentDoc.select(imgSelector);
|
|
1026
|
+
const imageExtensions = /\.(png|jpg|jpeg|gif|webp|svg|ico|bmp|tiff|avif)(\?.*)?$/i;
|
|
903
1027
|
const images = [];
|
|
904
|
-
|
|
1028
|
+
const scope = selector ? `${selector} ` : '';
|
|
1029
|
+
this.currentDoc.select(`${scope}img[src]`).each((el) => {
|
|
1030
|
+
const src = el.attr('src');
|
|
1031
|
+
if (src)
|
|
1032
|
+
images.push({ type: 'img', src, alt: el.attr('alt') });
|
|
1033
|
+
});
|
|
1034
|
+
this.currentDoc.select(`${scope}source[srcset]`).each((el) => {
|
|
1035
|
+
const srcset = el.attr('srcset');
|
|
1036
|
+
if (srcset) {
|
|
1037
|
+
const src = srcset.split(',')[0].trim().split(' ')[0];
|
|
1038
|
+
if (src)
|
|
1039
|
+
images.push({ type: 'source', src });
|
|
1040
|
+
}
|
|
1041
|
+
});
|
|
1042
|
+
this.currentDoc.select(`${scope}[style*="background"]`).each((el) => {
|
|
1043
|
+
const style = el.attr('style') || '';
|
|
1044
|
+
const matches = style.match(/url\(['"]?([^'"()]+)['"]?\)/gi);
|
|
1045
|
+
if (matches) {
|
|
1046
|
+
matches.forEach(m => {
|
|
1047
|
+
const src = m.replace(/url\(['"]?|['"]?\)/gi, '');
|
|
1048
|
+
if (imageExtensions.test(src))
|
|
1049
|
+
images.push({ type: 'bg', src });
|
|
1050
|
+
});
|
|
1051
|
+
}
|
|
1052
|
+
});
|
|
1053
|
+
if (!selector) {
|
|
1054
|
+
this.currentDoc.select('link[href]').each((el) => {
|
|
1055
|
+
const href = el.attr('href');
|
|
1056
|
+
if (href && imageExtensions.test(href)) {
|
|
1057
|
+
images.push({ type: 'link', src: href });
|
|
1058
|
+
}
|
|
1059
|
+
});
|
|
1060
|
+
this.currentDoc.select('meta[property="og:image"], meta[name="twitter:image"]').each((el) => {
|
|
1061
|
+
const content = el.attr('content');
|
|
1062
|
+
if (content)
|
|
1063
|
+
images.push({ type: 'meta', src: content });
|
|
1064
|
+
});
|
|
1065
|
+
}
|
|
1066
|
+
const uniqueImages = [...new Map(images.map(img => [img.src, img])).values()];
|
|
1067
|
+
uniqueImages.slice(0, 25).forEach((img, i) => {
|
|
1068
|
+
const typeLabel = colors.gray(`[${img.type}]`);
|
|
1069
|
+
const altText = img.alt ? colors.cyan(img.alt.slice(0, 25)) : '';
|
|
1070
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${typeLabel} ${altText} ${img.src.slice(0, 60)}`);
|
|
1071
|
+
});
|
|
1072
|
+
if (uniqueImages.length > 25) {
|
|
1073
|
+
console.log(colors.gray(` ... and ${uniqueImages.length - 25} more images`));
|
|
1074
|
+
}
|
|
1075
|
+
this.lastResponse = uniqueImages;
|
|
1076
|
+
console.log(colors.gray(`\n ${uniqueImages.length} image(s) found`));
|
|
1077
|
+
}
|
|
1078
|
+
catch (error) {
|
|
1079
|
+
console.error(colors.red(`Query failed: ${error.message}`));
|
|
1080
|
+
}
|
|
1081
|
+
console.log('');
|
|
1082
|
+
}
|
|
1083
|
+
async runSelectScripts() {
|
|
1084
|
+
if (!this.currentDoc) {
|
|
1085
|
+
console.log(colors.yellow('No document loaded. Use "scrap <url>" first.'));
|
|
1086
|
+
return;
|
|
1087
|
+
}
|
|
1088
|
+
try {
|
|
1089
|
+
const scripts = [];
|
|
1090
|
+
this.currentDoc.select('script[src]').each((el) => {
|
|
905
1091
|
const src = el.attr('src');
|
|
906
|
-
const alt = el.attr('alt') || '';
|
|
907
1092
|
if (src) {
|
|
908
|
-
|
|
1093
|
+
scripts.push({
|
|
1094
|
+
type: 'external',
|
|
1095
|
+
src,
|
|
1096
|
+
async: el.attr('async') !== undefined,
|
|
1097
|
+
defer: el.attr('defer') !== undefined
|
|
1098
|
+
});
|
|
1099
|
+
}
|
|
1100
|
+
});
|
|
1101
|
+
this.currentDoc.select('script:not([src])').each((el) => {
|
|
1102
|
+
const content = el.text();
|
|
1103
|
+
if (content.trim()) {
|
|
1104
|
+
scripts.push({
|
|
1105
|
+
type: 'inline',
|
|
1106
|
+
size: content.length
|
|
1107
|
+
});
|
|
1108
|
+
}
|
|
1109
|
+
});
|
|
1110
|
+
let extCount = 0, inlineCount = 0, totalInlineSize = 0;
|
|
1111
|
+
scripts.forEach((script, i) => {
|
|
1112
|
+
if (script.type === 'external') {
|
|
1113
|
+
extCount++;
|
|
1114
|
+
const flags = [
|
|
1115
|
+
script.async ? colors.cyan('async') : '',
|
|
1116
|
+
script.defer ? colors.cyan('defer') : ''
|
|
1117
|
+
].filter(Boolean).join(' ');
|
|
1118
|
+
if (i < 20) {
|
|
1119
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${colors.green('[ext]')} ${script.src?.slice(0, 70)} ${flags}`);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
else {
|
|
1123
|
+
inlineCount++;
|
|
1124
|
+
totalInlineSize += script.size || 0;
|
|
1125
|
+
if (i < 20) {
|
|
1126
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${colors.yellow('[inline]')} ${((script.size || 0) / 1024).toFixed(1)}kb`);
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
});
|
|
1130
|
+
if (scripts.length > 20) {
|
|
1131
|
+
console.log(colors.gray(` ... and ${scripts.length - 20} more scripts`));
|
|
1132
|
+
}
|
|
1133
|
+
this.lastResponse = scripts;
|
|
1134
|
+
console.log(colors.gray(`\n ${extCount} external, ${inlineCount} inline (${(totalInlineSize / 1024).toFixed(1)}kb total)`));
|
|
1135
|
+
}
|
|
1136
|
+
catch (error) {
|
|
1137
|
+
console.error(colors.red(`Query failed: ${error.message}`));
|
|
1138
|
+
}
|
|
1139
|
+
console.log('');
|
|
1140
|
+
}
|
|
1141
|
+
async runSelectCSS() {
|
|
1142
|
+
if (!this.currentDoc) {
|
|
1143
|
+
console.log(colors.yellow('No document loaded. Use "scrap <url>" first.'));
|
|
1144
|
+
return;
|
|
1145
|
+
}
|
|
1146
|
+
try {
|
|
1147
|
+
const styles = [];
|
|
1148
|
+
this.currentDoc.select('link[rel="stylesheet"]').each((el) => {
|
|
1149
|
+
const href = el.attr('href');
|
|
1150
|
+
if (href) {
|
|
1151
|
+
styles.push({
|
|
1152
|
+
type: 'external',
|
|
1153
|
+
href,
|
|
1154
|
+
media: el.attr('media')
|
|
1155
|
+
});
|
|
1156
|
+
}
|
|
1157
|
+
});
|
|
1158
|
+
this.currentDoc.select('style').each((el) => {
|
|
1159
|
+
const content = el.text();
|
|
1160
|
+
if (content.trim()) {
|
|
1161
|
+
styles.push({
|
|
1162
|
+
type: 'inline',
|
|
1163
|
+
size: content.length,
|
|
1164
|
+
media: el.attr('media')
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
});
|
|
1168
|
+
let extCount = 0, inlineCount = 0, totalInlineSize = 0;
|
|
1169
|
+
styles.forEach((style, i) => {
|
|
1170
|
+
if (style.type === 'external') {
|
|
1171
|
+
extCount++;
|
|
1172
|
+
const media = style.media ? colors.cyan(`[${style.media}]`) : '';
|
|
909
1173
|
if (i < 20) {
|
|
910
|
-
console.log(`${colors.gray(`${i + 1}.`)} ${colors.
|
|
1174
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${colors.green('[ext]')} ${style.href?.slice(0, 70)} ${media}`);
|
|
911
1175
|
}
|
|
912
1176
|
}
|
|
1177
|
+
else {
|
|
1178
|
+
inlineCount++;
|
|
1179
|
+
totalInlineSize += style.size || 0;
|
|
1180
|
+
const media = style.media ? colors.cyan(`[${style.media}]`) : '';
|
|
1181
|
+
if (i < 20) {
|
|
1182
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${colors.yellow('[inline]')} ${((style.size || 0) / 1024).toFixed(1)}kb ${media}`);
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
});
|
|
1186
|
+
if (styles.length > 20) {
|
|
1187
|
+
console.log(colors.gray(` ... and ${styles.length - 20} more stylesheets`));
|
|
1188
|
+
}
|
|
1189
|
+
this.lastResponse = styles;
|
|
1190
|
+
console.log(colors.gray(`\n ${extCount} external, ${inlineCount} inline (${(totalInlineSize / 1024).toFixed(1)}kb total)`));
|
|
1191
|
+
}
|
|
1192
|
+
catch (error) {
|
|
1193
|
+
console.error(colors.red(`Query failed: ${error.message}`));
|
|
1194
|
+
}
|
|
1195
|
+
console.log('');
|
|
1196
|
+
}
|
|
1197
|
+
async runSelectSourcemaps() {
|
|
1198
|
+
if (!this.currentDoc) {
|
|
1199
|
+
console.log(colors.yellow('No document loaded. Use "scrap <url>" first.'));
|
|
1200
|
+
return;
|
|
1201
|
+
}
|
|
1202
|
+
try {
|
|
1203
|
+
const sourcemaps = [];
|
|
1204
|
+
const sourceMappingURLPattern = /\/[/*]#\s*sourceMappingURL=([^\s*]+)/gi;
|
|
1205
|
+
this.currentDoc.select('script:not([src])').each((el) => {
|
|
1206
|
+
const content = el.text();
|
|
1207
|
+
const matches = content.matchAll(sourceMappingURLPattern);
|
|
1208
|
+
for (const match of matches) {
|
|
1209
|
+
sourcemaps.push({ type: 'inline-js', url: match[1] });
|
|
1210
|
+
}
|
|
1211
|
+
});
|
|
1212
|
+
this.currentDoc.select('style').each((el) => {
|
|
1213
|
+
const content = el.text();
|
|
1214
|
+
const matches = content.matchAll(sourceMappingURLPattern);
|
|
1215
|
+
for (const match of matches) {
|
|
1216
|
+
sourcemaps.push({ type: 'inline-css', url: match[1] });
|
|
1217
|
+
}
|
|
913
1218
|
});
|
|
914
|
-
|
|
915
|
-
|
|
1219
|
+
this.currentDoc.select('script[src]').each((el) => {
|
|
1220
|
+
const src = el.attr('src');
|
|
1221
|
+
if (src && !src.endsWith('.map')) {
|
|
1222
|
+
sourcemaps.push({ type: 'js-inferred', url: `${src}.map`, source: src });
|
|
1223
|
+
}
|
|
1224
|
+
});
|
|
1225
|
+
this.currentDoc.select('link[rel="stylesheet"]').each((el) => {
|
|
1226
|
+
const href = el.attr('href');
|
|
1227
|
+
if (href && !href.endsWith('.map')) {
|
|
1228
|
+
sourcemaps.push({ type: 'css-inferred', url: `${href}.map`, source: href });
|
|
1229
|
+
}
|
|
1230
|
+
});
|
|
1231
|
+
this.currentDoc.select('script[src$=".map"], link[href$=".map"]').each((el) => {
|
|
1232
|
+
const url = el.attr('src') || el.attr('href');
|
|
1233
|
+
if (url)
|
|
1234
|
+
sourcemaps.push({ type: 'direct', url });
|
|
1235
|
+
});
|
|
1236
|
+
const uniqueMaps = [...new Map(sourcemaps.map(m => [m.url, m])).values()];
|
|
1237
|
+
const confirmed = uniqueMaps.filter(m => !m.type.includes('inferred'));
|
|
1238
|
+
const inferred = uniqueMaps.filter(m => m.type.includes('inferred'));
|
|
1239
|
+
if (confirmed.length > 0) {
|
|
1240
|
+
console.log(colors.green('Confirmed sourcemaps:'));
|
|
1241
|
+
confirmed.forEach((m, i) => {
|
|
1242
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${colors.cyan(`[${m.type}]`)} ${m.url}`);
|
|
1243
|
+
});
|
|
1244
|
+
}
|
|
1245
|
+
if (inferred.length > 0) {
|
|
1246
|
+
console.log(colors.yellow('\nPotential sourcemaps (inferred):'));
|
|
1247
|
+
inferred.slice(0, 15).forEach((m, i) => {
|
|
1248
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${colors.gray(`[${m.type}]`)} ${m.url.slice(0, 70)}`);
|
|
1249
|
+
});
|
|
1250
|
+
if (inferred.length > 15) {
|
|
1251
|
+
console.log(colors.gray(` ... and ${inferred.length - 15} more`));
|
|
1252
|
+
}
|
|
916
1253
|
}
|
|
917
|
-
this.lastResponse =
|
|
918
|
-
console.log(colors.gray(`\n ${
|
|
1254
|
+
this.lastResponse = uniqueMaps;
|
|
1255
|
+
console.log(colors.gray(`\n ${confirmed.length} confirmed, ${inferred.length} inferred sourcemap(s)`));
|
|
1256
|
+
console.log(colors.gray(` Use $unmap <url> to extract original sources`));
|
|
919
1257
|
}
|
|
920
1258
|
catch (error) {
|
|
921
1259
|
console.error(colors.red(`Query failed: ${error.message}`));
|
|
922
1260
|
}
|
|
923
1261
|
console.log('');
|
|
924
1262
|
}
|
|
1263
|
+
async runUnmap(urlArg) {
|
|
1264
|
+
let mapUrl = urlArg;
|
|
1265
|
+
if (!mapUrl && Array.isArray(this.lastResponse)) {
|
|
1266
|
+
const maps = this.lastResponse;
|
|
1267
|
+
const confirmed = maps.filter(m => !m.type.includes('inferred'));
|
|
1268
|
+
if (confirmed.length > 0) {
|
|
1269
|
+
mapUrl = confirmed[0].url;
|
|
1270
|
+
console.log(colors.gray(`Using: ${mapUrl}`));
|
|
1271
|
+
}
|
|
1272
|
+
else if (maps.length > 0) {
|
|
1273
|
+
mapUrl = maps[0].url;
|
|
1274
|
+
console.log(colors.gray(`Using (inferred): ${mapUrl}`));
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
if (!mapUrl) {
|
|
1278
|
+
console.log(colors.yellow('Usage: $unmap <sourcemap-url>'));
|
|
1279
|
+
console.log(colors.gray(' Or run $sourcemaps first to find sourcemaps'));
|
|
1280
|
+
return;
|
|
1281
|
+
}
|
|
1282
|
+
if (!mapUrl.startsWith('http') && this.baseUrl) {
|
|
1283
|
+
const base = new URL(this.baseUrl);
|
|
1284
|
+
mapUrl = new URL(mapUrl, base).toString();
|
|
1285
|
+
}
|
|
1286
|
+
console.log(colors.cyan(`Fetching sourcemap: ${mapUrl}`));
|
|
1287
|
+
try {
|
|
1288
|
+
const response = await this.client.get(mapUrl);
|
|
1289
|
+
const mapData = await response.json();
|
|
1290
|
+
if (!mapData.sources || !Array.isArray(mapData.sources)) {
|
|
1291
|
+
console.log(colors.red('Invalid sourcemap: missing sources array'));
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
console.log(colors.green(`\nSourcemap v${mapData.version || '?'}`));
|
|
1295
|
+
if (mapData.file)
|
|
1296
|
+
console.log(colors.gray(` File: ${mapData.file}`));
|
|
1297
|
+
if (mapData.sourceRoot)
|
|
1298
|
+
console.log(colors.gray(` Root: ${mapData.sourceRoot}`));
|
|
1299
|
+
console.log(colors.gray(` Sources: ${mapData.sources.length}`));
|
|
1300
|
+
if (mapData.names)
|
|
1301
|
+
console.log(colors.gray(` Names: ${mapData.names.length}`));
|
|
1302
|
+
console.log(colors.bold('\nOriginal sources:'));
|
|
1303
|
+
mapData.sources.forEach((source, i) => {
|
|
1304
|
+
const hasContent = mapData.sourcesContent && mapData.sourcesContent[i];
|
|
1305
|
+
const sizeInfo = hasContent
|
|
1306
|
+
? colors.green(`[${(mapData.sourcesContent[i].length / 1024).toFixed(1)}kb]`)
|
|
1307
|
+
: colors.yellow('[no content]');
|
|
1308
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${sizeInfo} ${source}`);
|
|
1309
|
+
});
|
|
1310
|
+
this.lastResponse = {
|
|
1311
|
+
url: mapUrl,
|
|
1312
|
+
data: mapData,
|
|
1313
|
+
sources: mapData.sources.map((source, i) => ({
|
|
1314
|
+
path: source,
|
|
1315
|
+
content: mapData.sourcesContent?.[i] || null
|
|
1316
|
+
}))
|
|
1317
|
+
};
|
|
1318
|
+
const withContent = mapData.sourcesContent?.filter(c => c).length || 0;
|
|
1319
|
+
console.log(colors.gray(`\n ${withContent}/${mapData.sources.length} sources have embedded content`));
|
|
1320
|
+
if (withContent > 0) {
|
|
1321
|
+
console.log(colors.gray(` Use $unmap:view <index> to view source content`));
|
|
1322
|
+
console.log(colors.gray(` Use $unmap:save <dir> to save all sources to disk`));
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
catch (error) {
|
|
1326
|
+
if (error.status === 404) {
|
|
1327
|
+
console.log(colors.yellow(`Sourcemap not found (404): ${mapUrl}`));
|
|
1328
|
+
}
|
|
1329
|
+
else {
|
|
1330
|
+
console.error(colors.red(`Failed to fetch sourcemap: ${error.message}`));
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
console.log('');
|
|
1334
|
+
}
|
|
1335
|
+
async runUnmapView(indexStr) {
|
|
1336
|
+
if (!this.lastResponse || !this.lastResponse.sources) {
|
|
1337
|
+
console.log(colors.yellow('No sourcemap loaded. Use $unmap <url> first.'));
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1340
|
+
const index = parseInt(indexStr, 10) - 1;
|
|
1341
|
+
const sources = this.lastResponse.sources;
|
|
1342
|
+
if (isNaN(index) || index < 0 || index >= sources.length) {
|
|
1343
|
+
console.log(colors.yellow(`Invalid index. Use 1-${sources.length}`));
|
|
1344
|
+
return;
|
|
1345
|
+
}
|
|
1346
|
+
const source = sources[index];
|
|
1347
|
+
if (!source.content) {
|
|
1348
|
+
console.log(colors.yellow(`No embedded content for: ${source.path}`));
|
|
1349
|
+
return;
|
|
1350
|
+
}
|
|
1351
|
+
console.log(colors.bold(`\n─── ${source.path} ───\n`));
|
|
1352
|
+
const ext = source.path.split('.').pop()?.toLowerCase();
|
|
1353
|
+
if (['js', 'ts', 'jsx', 'tsx', 'mjs', 'cjs'].includes(ext || '')) {
|
|
1354
|
+
try {
|
|
1355
|
+
console.log(highlight(source.content, { linenos: true }));
|
|
1356
|
+
}
|
|
1357
|
+
catch {
|
|
1358
|
+
console.log(source.content);
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
else {
|
|
1362
|
+
console.log(source.content);
|
|
1363
|
+
}
|
|
1364
|
+
console.log(colors.bold(`\n─── end ───\n`));
|
|
1365
|
+
}
|
|
1366
|
+
async runUnmapSave(dir) {
|
|
1367
|
+
if (!this.lastResponse || !this.lastResponse.sources) {
|
|
1368
|
+
console.log(colors.yellow('No sourcemap loaded. Use $unmap <url> first.'));
|
|
1369
|
+
return;
|
|
1370
|
+
}
|
|
1371
|
+
const outputDir = dir || './sourcemap-extracted';
|
|
1372
|
+
const sources = this.lastResponse.sources;
|
|
1373
|
+
const { promises: fs } = await import('node:fs');
|
|
1374
|
+
const path = await import('node:path');
|
|
1375
|
+
let saved = 0, skipped = 0;
|
|
1376
|
+
for (const source of sources) {
|
|
1377
|
+
if (!source.content) {
|
|
1378
|
+
skipped++;
|
|
1379
|
+
continue;
|
|
1380
|
+
}
|
|
1381
|
+
let cleanPath = source.path
|
|
1382
|
+
.replace(/^webpack:\/\/[^/]*\//, '')
|
|
1383
|
+
.replace(/^\.*\//, '')
|
|
1384
|
+
.replace(/^node_modules\//, 'node_modules/');
|
|
1385
|
+
const fullPath = path.join(outputDir, cleanPath);
|
|
1386
|
+
const dirname = path.dirname(fullPath);
|
|
1387
|
+
try {
|
|
1388
|
+
await fs.mkdir(dirname, { recursive: true });
|
|
1389
|
+
await fs.writeFile(fullPath, source.content, 'utf-8');
|
|
1390
|
+
saved++;
|
|
1391
|
+
console.log(colors.green(` ✓ ${cleanPath}`));
|
|
1392
|
+
}
|
|
1393
|
+
catch (err) {
|
|
1394
|
+
console.log(colors.red(` ✗ ${cleanPath}: ${err.message}`));
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
console.log(colors.gray(`\n Saved ${saved} files to ${outputDir}`));
|
|
1398
|
+
if (skipped > 0) {
|
|
1399
|
+
console.log(colors.yellow(` Skipped ${skipped} sources without embedded content`));
|
|
1400
|
+
}
|
|
1401
|
+
console.log('');
|
|
1402
|
+
}
|
|
1403
|
+
async runBeautify(urlArg) {
|
|
1404
|
+
if (!urlArg) {
|
|
1405
|
+
console.log(colors.yellow('Usage: $beautify <url-to-js-or-css>'));
|
|
1406
|
+
console.log(colors.gray(' Downloads and formats minified JS/CSS code'));
|
|
1407
|
+
return;
|
|
1408
|
+
}
|
|
1409
|
+
let url = urlArg;
|
|
1410
|
+
if (!url.startsWith('http') && this.baseUrl) {
|
|
1411
|
+
const base = new URL(this.baseUrl);
|
|
1412
|
+
url = new URL(url, base).toString();
|
|
1413
|
+
}
|
|
1414
|
+
console.log(colors.cyan(`Fetching: ${url}`));
|
|
1415
|
+
try {
|
|
1416
|
+
const response = await this.client.get(url);
|
|
1417
|
+
const code = await response.text();
|
|
1418
|
+
const isCSS = url.endsWith('.css') || response.headers.get('content-type')?.includes('css');
|
|
1419
|
+
console.log(colors.gray(` Size: ${(code.length / 1024).toFixed(1)}kb`));
|
|
1420
|
+
const formatted = isCSS ? this.beautifyCSS(code) : this.beautifyJS(code);
|
|
1421
|
+
console.log(colors.bold(`\n─── Beautified ${isCSS ? 'CSS' : 'JS'} ───\n`));
|
|
1422
|
+
try {
|
|
1423
|
+
console.log(highlight(formatted, { linenos: true }));
|
|
1424
|
+
}
|
|
1425
|
+
catch {
|
|
1426
|
+
console.log(formatted);
|
|
1427
|
+
}
|
|
1428
|
+
console.log(colors.bold(`\n─── end ───`));
|
|
1429
|
+
this.lastResponse = { url, original: code, formatted, type: isCSS ? 'css' : 'js' };
|
|
1430
|
+
console.log(colors.gray(`\n Use $beautify:save <file> to save formatted code`));
|
|
1431
|
+
}
|
|
1432
|
+
catch (error) {
|
|
1433
|
+
console.error(colors.red(`Failed to fetch: ${error.message}`));
|
|
1434
|
+
}
|
|
1435
|
+
console.log('');
|
|
1436
|
+
}
|
|
1437
|
+
beautifyJS(code) {
|
|
1438
|
+
let result = '';
|
|
1439
|
+
let indent = 0;
|
|
1440
|
+
let inString = null;
|
|
1441
|
+
let inComment = false;
|
|
1442
|
+
let inLineComment = false;
|
|
1443
|
+
let i = 0;
|
|
1444
|
+
const addNewline = () => {
|
|
1445
|
+
result += '\n' + ' '.repeat(indent);
|
|
1446
|
+
};
|
|
1447
|
+
while (i < code.length) {
|
|
1448
|
+
const char = code[i];
|
|
1449
|
+
const next = code[i + 1];
|
|
1450
|
+
const prev = code[i - 1];
|
|
1451
|
+
if (!inComment && !inLineComment) {
|
|
1452
|
+
if ((char === '"' || char === "'" || char === '`') && prev !== '\\') {
|
|
1453
|
+
if (inString === char) {
|
|
1454
|
+
inString = null;
|
|
1455
|
+
}
|
|
1456
|
+
else if (!inString) {
|
|
1457
|
+
inString = char;
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
if (!inString && !inComment && !inLineComment) {
|
|
1462
|
+
if (char === '/' && next === '*') {
|
|
1463
|
+
inComment = true;
|
|
1464
|
+
result += char;
|
|
1465
|
+
i++;
|
|
1466
|
+
continue;
|
|
1467
|
+
}
|
|
1468
|
+
if (char === '/' && next === '/') {
|
|
1469
|
+
inLineComment = true;
|
|
1470
|
+
result += char;
|
|
1471
|
+
i++;
|
|
1472
|
+
continue;
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
if (inComment && char === '*' && next === '/') {
|
|
1476
|
+
result += '*/';
|
|
1477
|
+
inComment = false;
|
|
1478
|
+
i += 2;
|
|
1479
|
+
continue;
|
|
1480
|
+
}
|
|
1481
|
+
if (inLineComment && char === '\n') {
|
|
1482
|
+
inLineComment = false;
|
|
1483
|
+
}
|
|
1484
|
+
if (inString || inComment || inLineComment) {
|
|
1485
|
+
result += char;
|
|
1486
|
+
i++;
|
|
1487
|
+
continue;
|
|
1488
|
+
}
|
|
1489
|
+
if (char === '{') {
|
|
1490
|
+
result += ' {';
|
|
1491
|
+
indent++;
|
|
1492
|
+
addNewline();
|
|
1493
|
+
i++;
|
|
1494
|
+
continue;
|
|
1495
|
+
}
|
|
1496
|
+
if (char === '}') {
|
|
1497
|
+
indent = Math.max(0, indent - 1);
|
|
1498
|
+
addNewline();
|
|
1499
|
+
result += '}';
|
|
1500
|
+
if (next && next !== ';' && next !== ',' && next !== ')' && next !== '\n') {
|
|
1501
|
+
addNewline();
|
|
1502
|
+
}
|
|
1503
|
+
i++;
|
|
1504
|
+
continue;
|
|
1505
|
+
}
|
|
1506
|
+
if (char === ';') {
|
|
1507
|
+
result += ';';
|
|
1508
|
+
if (next && next !== '}' && next !== '\n') {
|
|
1509
|
+
addNewline();
|
|
1510
|
+
}
|
|
1511
|
+
i++;
|
|
1512
|
+
continue;
|
|
1513
|
+
}
|
|
1514
|
+
if (char === ' ' || char === '\t' || char === '\n' || char === '\r') {
|
|
1515
|
+
if (result.length > 0 && !/\s$/.test(result)) {
|
|
1516
|
+
result += ' ';
|
|
1517
|
+
}
|
|
1518
|
+
i++;
|
|
1519
|
+
continue;
|
|
1520
|
+
}
|
|
1521
|
+
result += char;
|
|
1522
|
+
i++;
|
|
1523
|
+
}
|
|
1524
|
+
return result.trim();
|
|
1525
|
+
}
|
|
1526
|
+
beautifyCSS(code) {
|
|
1527
|
+
let result = '';
|
|
1528
|
+
let indent = 0;
|
|
1529
|
+
let inString = null;
|
|
1530
|
+
let i = 0;
|
|
1531
|
+
const addNewline = () => {
|
|
1532
|
+
result += '\n' + ' '.repeat(indent);
|
|
1533
|
+
};
|
|
1534
|
+
while (i < code.length) {
|
|
1535
|
+
const char = code[i];
|
|
1536
|
+
const next = code[i + 1];
|
|
1537
|
+
const prev = code[i - 1];
|
|
1538
|
+
if ((char === '"' || char === "'") && prev !== '\\') {
|
|
1539
|
+
if (inString === char) {
|
|
1540
|
+
inString = null;
|
|
1541
|
+
}
|
|
1542
|
+
else if (!inString) {
|
|
1543
|
+
inString = char;
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
if (inString) {
|
|
1547
|
+
result += char;
|
|
1548
|
+
i++;
|
|
1549
|
+
continue;
|
|
1550
|
+
}
|
|
1551
|
+
if (char === '{') {
|
|
1552
|
+
result += ' {';
|
|
1553
|
+
indent++;
|
|
1554
|
+
addNewline();
|
|
1555
|
+
i++;
|
|
1556
|
+
continue;
|
|
1557
|
+
}
|
|
1558
|
+
if (char === '}') {
|
|
1559
|
+
indent = Math.max(0, indent - 1);
|
|
1560
|
+
addNewline();
|
|
1561
|
+
result += '}';
|
|
1562
|
+
addNewline();
|
|
1563
|
+
i++;
|
|
1564
|
+
continue;
|
|
1565
|
+
}
|
|
1566
|
+
if (char === ';') {
|
|
1567
|
+
result += ';';
|
|
1568
|
+
addNewline();
|
|
1569
|
+
i++;
|
|
1570
|
+
continue;
|
|
1571
|
+
}
|
|
1572
|
+
if (char === ',' && indent === 0) {
|
|
1573
|
+
result += ',';
|
|
1574
|
+
addNewline();
|
|
1575
|
+
i++;
|
|
1576
|
+
continue;
|
|
1577
|
+
}
|
|
1578
|
+
if (char === ' ' || char === '\t' || char === '\n' || char === '\r') {
|
|
1579
|
+
if (result.length > 0 && !/\s$/.test(result)) {
|
|
1580
|
+
result += ' ';
|
|
1581
|
+
}
|
|
1582
|
+
i++;
|
|
1583
|
+
continue;
|
|
1584
|
+
}
|
|
1585
|
+
result += char;
|
|
1586
|
+
i++;
|
|
1587
|
+
}
|
|
1588
|
+
return result.trim();
|
|
1589
|
+
}
|
|
1590
|
+
async runBeautifySave(filename) {
|
|
1591
|
+
if (!this.lastResponse || !this.lastResponse.formatted) {
|
|
1592
|
+
console.log(colors.yellow('No beautified code. Use $beautify <url> first.'));
|
|
1593
|
+
return;
|
|
1594
|
+
}
|
|
1595
|
+
const outputFile = filename || `beautified.${this.lastResponse.type}`;
|
|
1596
|
+
const { promises: fs } = await import('node:fs');
|
|
1597
|
+
try {
|
|
1598
|
+
await fs.writeFile(outputFile, this.lastResponse.formatted, 'utf-8');
|
|
1599
|
+
console.log(colors.green(` ✓ Saved to ${outputFile}`));
|
|
1600
|
+
}
|
|
1601
|
+
catch (err) {
|
|
1602
|
+
console.log(colors.red(` ✗ Failed to save: ${err.message}`));
|
|
1603
|
+
}
|
|
1604
|
+
console.log('');
|
|
1605
|
+
}
|
|
925
1606
|
async runSelectTable(selector) {
|
|
926
1607
|
if (!this.currentDoc) {
|
|
927
1608
|
console.log(colors.yellow('No document loaded. Use "scrap <url>" first.'));
|
|
@@ -966,7 +1647,8 @@ export class RekShell {
|
|
|
966
1647
|
${colors.bold('Core Commands:')}
|
|
967
1648
|
${colors.green('url <url>')} Set persistent Base URL.
|
|
968
1649
|
${colors.green('set <key>=<val>')} Set a session variable.
|
|
969
|
-
${colors.green('vars')} List all session variables.
|
|
1650
|
+
${colors.green('vars')} List all session and env variables.
|
|
1651
|
+
${colors.green('env [path]')} Load .env file (default: ./.env).
|
|
970
1652
|
${colors.green('clear')} Clear the screen.
|
|
971
1653
|
${colors.green('exit')} Exit the console.
|
|
972
1654
|
|
|
@@ -1005,7 +1687,15 @@ export class RekShell {
|
|
|
1005
1687
|
${colors.green('$attr <name> <sel>')} Extract attribute values.
|
|
1006
1688
|
${colors.green('$html <selector>')} Get inner HTML.
|
|
1007
1689
|
${colors.green('$links [selector]')} List all links.
|
|
1008
|
-
${colors.green('$images [selector]')} List all images.
|
|
1690
|
+
${colors.green('$images [selector]')} List all images (img, bg, og:image, favicon).
|
|
1691
|
+
${colors.green('$scripts')} List all scripts (external + inline).
|
|
1692
|
+
${colors.green('$css')} List all stylesheets (external + inline).
|
|
1693
|
+
${colors.green('$sourcemaps')} Find sourcemaps (confirmed + inferred).
|
|
1694
|
+
${colors.green('$unmap <url>')} Download and parse sourcemap.
|
|
1695
|
+
${colors.green('$unmap:view <n>')} View source file by index.
|
|
1696
|
+
${colors.green('$unmap:save [dir]')} Save all sources to disk.
|
|
1697
|
+
${colors.green('$beautify <url>')} Format minified JS/CSS code.
|
|
1698
|
+
${colors.green('$beautify:save [f]')} Save beautified code to file.
|
|
1009
1699
|
${colors.green('$table <selector>')} Extract table as data.
|
|
1010
1700
|
|
|
1011
1701
|
${colors.bold('Examples:')}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "recker",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.10",
|
|
4
4
|
"description": "AI & DevX focused HTTP client for Node.js 18+",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -114,6 +114,7 @@
|
|
|
114
114
|
"access": "public"
|
|
115
115
|
},
|
|
116
116
|
"dependencies": {
|
|
117
|
+
"cheerio": "^1.0.0",
|
|
117
118
|
"commander": "^12.0.0",
|
|
118
119
|
"ora": "^8.0.0",
|
|
119
120
|
"undici": "^7.16.0",
|
|
@@ -121,7 +122,6 @@
|
|
|
121
122
|
},
|
|
122
123
|
"peerDependencies": {
|
|
123
124
|
"cardinal": "^2.1.0",
|
|
124
|
-
"cheerio": "^1.0.0",
|
|
125
125
|
"ioredis": "^5.0.0",
|
|
126
126
|
"ssh2-sftp-client": "^11.0.0"
|
|
127
127
|
},
|
|
@@ -129,9 +129,6 @@
|
|
|
129
129
|
"cardinal": {
|
|
130
130
|
"optional": true
|
|
131
131
|
},
|
|
132
|
-
"cheerio": {
|
|
133
|
-
"optional": true
|
|
134
|
-
},
|
|
135
132
|
"ioredis": {
|
|
136
133
|
"optional": true
|
|
137
134
|
},
|
|
@@ -149,7 +146,6 @@
|
|
|
149
146
|
"cardinal": "^2.1.1",
|
|
150
147
|
"cheerio": "^1.0.0",
|
|
151
148
|
"commander": "^14.0.0",
|
|
152
|
-
"docsify-cli": "^4.4.4",
|
|
153
149
|
"domhandler": "^5.0.3",
|
|
154
150
|
"got": "^14.6.5",
|
|
155
151
|
"husky": "^9.1.7",
|
|
@@ -158,6 +154,7 @@
|
|
|
158
154
|
"needle": "^3.3.1",
|
|
159
155
|
"ora": "^9.0.0",
|
|
160
156
|
"picocolors": "^1.1.1",
|
|
157
|
+
"serve": "^14.2.5",
|
|
161
158
|
"ssh2-sftp-client": "^12.0.1",
|
|
162
159
|
"superagent": "^10.2.3",
|
|
163
160
|
"tsx": "^4.20.6",
|
|
@@ -171,7 +168,7 @@
|
|
|
171
168
|
"test:coverage": "vitest run --coverage",
|
|
172
169
|
"bench": "tsx benchmark/index.ts",
|
|
173
170
|
"bench:all": "tsx benchmark/run-all.ts",
|
|
174
|
-
"docs": "
|
|
171
|
+
"docs": "serve docs -p 3000 -o",
|
|
175
172
|
"lint": "echo \"No linting configured for this project.\" && exit 0",
|
|
176
173
|
"cli": "tsx src/cli/index.ts"
|
|
177
174
|
}
|