recker 1.0.27 → 1.0.28-next.32fe8ef

package/dist/browser/scrape/extractors.js

@@ -76,6 +76,7 @@ export function extractImages($, options) {
             height: height ? parseInt(height, 10) : undefined,
             srcset: $el.attr('srcset'),
             loading: $el.attr('loading'),
+            decoding: $el.attr('decoding'),
         });
     });
     return images;
@@ -117,7 +118,7 @@ export function extractMeta($) {
                 meta.author = content;
                 break;
             case 'robots':
-                meta.robots = content;
+                meta.robots = content.split(',').map((r) => r.trim().toLowerCase());
                 break;
             case 'viewport':
                 meta.viewport = content;

package/dist/browser/scrape/types.d.ts

@@ -14,13 +14,14 @@ export interface ExtractedImage {
     height?: number;
     srcset?: string;
     loading?: 'lazy' | 'eager';
+    decoding?: 'async' | 'auto' | 'sync';
 }
 export interface ExtractedMeta {
     title?: string;
     description?: string;
     keywords?: string[];
     author?: string;
-    robots?: string;
+    robots?: string[];
     canonical?: string;
     viewport?: string;
     charset?: string;

package/dist/cli/index.js

@@ -3,6 +3,7 @@ import { program } from 'commander';
 import { promises as fs } from 'node:fs';
 import { join } from 'node:path';
 import colors from '../utils/colors.js';
+import { formatColumns } from '../utils/columns.js';
 async function readStdin() {
     if (process.stdin.isTTY) {
         return null;
@@ -120,7 +121,13 @@ async function main() {
         }
         return { method, url, headers, data };
     }
-    const PRESET_NAMES = Object.keys(presets).filter(k => k !== 'registry' && !k.startsWith('_'));
+    const utilityFunctions = [
+        'registry', 'presetRegistry', 'detectPreset', 'getPreset',
+        'listPresets', 'listAIPresets', 'listCloudPresets', 'listSaaSPresets', 'listDevToolsPresets'
+    ];
+    const PRESET_NAMES = Object.keys(presets)
+        .filter(k => !utilityFunctions.includes(k) && !k.startsWith('_') && typeof presets[k] === 'function')
+        .sort();
     program
         .name('rek')
         .description('The HTTP Client for Humans (and Robots)')
@@ -131,7 +138,7 @@ async function main() {
         .option('-o, --output <file>', 'Write response body to file')
         .option('-j, --json', 'Force JSON content-type')
         .option('-e, --env [path]', 'Load .env file from current directory or specified path')
-        .addHelpText('after', `
+        .addHelpText('after', () => `
 ${colors.bold(colors.yellow('Examples:'))}
   ${colors.green('$ rek httpbin.org/json')}
   ${colors.green('$ rek post api.com/users name="Cyber" role="Admin"')}
@@ -139,7 +146,7 @@ ${colors.bold(colors.yellow('Examples:'))}
   ${colors.green('$ rek @openai/v1/chat/completions model="gpt-5.1"')}
 
 ${colors.bold(colors.yellow('Available Presets:'))}
-  ${colors.cyan(PRESET_NAMES.map(p => '@' + p).join(', '))}
+${formatColumns(PRESET_NAMES, { prefix: '@', indent: 2, minWidth: 16, transform: colors.cyan })}
 `)
         .action(async (args, options) => {
         if (args.length === 0) {
@@ -381,6 +388,138 @@ ${colors.bold('Details:')}`);
             process.exit(1);
         }
     });
+    program
+        .command('seo')
+        .description('Analyze page SEO (title, meta, headings, links, images, structured data)')
+        .argument('<url>', 'URL to analyze')
+        .option('-a, --all', 'Show all checks including passed ones')
+        .option('--format <format>', 'Output format: text (default) or json', 'text')
+        .addHelpText('after', `
+${colors.bold(colors.yellow('Examples:'))}
+  ${colors.green('$ rek seo example.com')}                    ${colors.gray('Basic SEO analysis')}
+  ${colors.green('$ rek seo example.com -a')}                 ${colors.gray('Show all checks')}
+  ${colors.green('$ rek seo example.com --format json')}      ${colors.gray('Output as JSON')}
+  ${colors.green('$ rek seo example.com --format json | jq')} ${colors.gray('Pipe to jq for processing')}
+
+${colors.bold(colors.yellow('Checks:'))}
+  ${colors.cyan('Title Tag')}          Length and presence
+  ${colors.cyan('Meta Description')}   Length and presence
+  ${colors.cyan('Headings')}           H1 presence and hierarchy
+  ${colors.cyan('Images')}             Alt text coverage
+  ${colors.cyan('Links')}              Internal/external distribution
+  ${colors.cyan('OpenGraph')}          Social sharing meta tags
+  ${colors.cyan('Twitter Card')}       Twitter sharing meta tags
+  ${colors.cyan('Structured Data')}    JSON-LD presence
+  ${colors.cyan('Technical')}          Canonical, viewport, lang
+`)
+        .action(async (url, options) => {
+        if (!url.startsWith('http'))
+            url = `https://${url}`;
+        const isJson = options.format === 'json';
+        const { createClient } = await import('../core/client.js');
+        const { analyzeSeo } = await import('../seo/analyzer.js');
+        if (!isJson) {
+            console.log(colors.gray(`Analyzing SEO for ${url}...`));
+        }
+        try {
+            const client = createClient({ timeout: 30000 });
+            const res = await client.get(url);
+            const html = await res.text();
+            const report = await analyzeSeo(html, { baseUrl: url });
+            if (isJson) {
+                const jsonOutput = {
+                    url,
+                    analyzedAt: new Date().toISOString(),
+                    score: report.score,
+                    grade: report.grade,
+                    title: report.title,
+                    metaDescription: report.metaDescription,
+                    content: report.content,
+                    headings: report.headings,
+                    links: report.links,
+                    images: report.images,
+                    openGraph: report.social.openGraph,
+                    twitterCard: report.social.twitterCard,
+                    jsonLd: report.jsonLd,
+                    technical: report.technical,
+                    checks: report.checks,
+                    summary: {
+                        total: report.checks.length,
+                        passed: report.checks.filter(c => c.status === 'pass').length,
+                        warnings: report.checks.filter(c => c.status === 'warn').length,
+                        errors: report.checks.filter(c => c.status === 'fail').length,
+                        info: report.checks.filter(c => c.status === 'info').length,
+                    },
+                };
+                console.log(JSON.stringify(jsonOutput, null, 2));
+                return;
+            }
+            let gradeColor = colors.red;
+            if (report.grade === 'A')
+                gradeColor = colors.green;
+            else if (report.grade === 'B')
+                gradeColor = colors.blue;
+            else if (report.grade === 'C')
+                gradeColor = colors.yellow;
+            console.log(`
+${colors.bold(colors.cyan('🔍 SEO Analysis Report'))}
+${colors.gray('URL:')} ${url}
+${colors.gray('Grade:')} ${gradeColor(colors.bold(report.grade))}  ${colors.gray('Score:')} ${report.score}/100
+`);
+            if (report.title) {
+                console.log(`${colors.bold('Title:')} ${colors.gray(report.title.text.slice(0, 60))}${report.title.text.length > 60 ? '...' : ''} ${colors.gray(`(${report.title.length} chars)`)}`);
+            }
+            if (report.metaDescription) {
+                console.log(`${colors.bold('Description:')} ${colors.gray(report.metaDescription.text.slice(0, 80))}${report.metaDescription.text.length > 80 ? '...' : ''}`);
+            }
+            console.log('');
+            console.log(`${colors.bold('Content Metrics:')}`);
+            console.log(`  ${colors.gray('Words:')} ${report.content.wordCount}  ${colors.gray('Reading time:')} ~${report.content.readingTimeMinutes} min`);
+            console.log(`  ${colors.gray('Headings:')} H1×${report.headings.h1Count}, total ${report.headings.structure.length}`);
+            console.log(`  ${colors.gray('Links:')} ${report.links.total} (${report.links.internal} internal, ${report.links.external} external)`);
+            console.log(`  ${colors.gray('Images:')} ${report.images.total} (${report.images.withAlt} with alt, ${report.images.withoutAlt} without)`);
+            console.log('');
+            console.log(`${colors.bold('Checks:')}`);
+            const checksToShow = options.all
+                ? report.checks
+                : report.checks.filter(c => c.status !== 'pass' && c.status !== 'info');
+            if (checksToShow.length === 0 && !options.all) {
+                console.log(colors.green('  All checks passed! Use -a to see details.'));
+            }
+            else {
+                for (const check of checksToShow) {
+                    const icon = check.status === 'pass' ? colors.green('✔')
+                        : check.status === 'warn' ? colors.yellow('⚠')
+                            : check.status === 'fail' ? colors.red('✖')
+                                : colors.gray('ℹ');
+                    const name = colors.bold(check.name.padEnd(18));
+                    console.log(`  ${icon} ${name} ${check.message}`);
+                    if (check.recommendation && check.status !== 'pass') {
+                        console.log(`      ${colors.gray('→')} ${colors.gray(check.recommendation)}`);
+                    }
+                    const evidence = check.evidence;
+                    if (evidence && check.status !== 'pass') {
+                        if (evidence.found && Array.isArray(evidence.found) && evidence.found.length > 0) {
+                            const items = evidence.found.slice(0, 3);
+                            console.log(`      ${colors.gray('Found:')} ${colors.red(items.join(', '))}${evidence.found.length > 3 ? ` (+${evidence.found.length - 3} more)` : ''}`);
+                        }
+                        if (evidence.example) {
+                            console.log(`      ${colors.gray('Example:')} ${colors.cyan(evidence.example.split('\n')[0])}`);
+                        }
+                    }
+                }
+            }
+            console.log('');
+            if (report.jsonLd.count > 0) {
+                console.log(`${colors.bold('Structured Data:')} ${report.jsonLd.types.join(', ') || 'Present'}`);
+                console.log('');
+            }
+        }
+        catch (error) {
+            console.error(colors.red(`SEO analysis failed: ${error.message}`));
+            process.exit(1);
+        }
+    });
     program
         .command('ip')
         .description('Get IP address intelligence using local GeoLite2 database')

package/dist/cli/tui/shell.d.ts

@@ -43,6 +43,7 @@ export declare class RekShell {
     private runWhois;
     private runTLS;
     private runSecurityGrader;
+    private runSeo;
     private runIpIntelligence;
     private runDNS;
     private runDNSPropagation;

package/dist/cli/tui/shell.js

@@ -14,6 +14,7 @@ import colors from '../../utils/colors.js';
 import { getShellSearch } from './shell-search.js';
 import { openSearchPanel } from './search-panel.js';
 import { ScrollBuffer, parseScrollKey, parseMouseScroll, disableMouseReporting } from './scroll-buffer.js';
+import { analyzeSeo } from '../../seo/index.js';
 let highlight;
 async function initDependencies() {
     if (!highlight) {
@@ -343,6 +344,9 @@ export class RekShell {
             case 'security':
                 await this.runSecurityGrader(parts[1]);
                 return;
+            case 'seo':
+                await this.runSeo(parts[1], parts.includes('-a') || parts.includes('--all'), parts.includes('--format') && parts[parts.indexOf('--format') + 1] === 'json');
+                return;
             case 'ip':
                 await this.runIpIntelligence(parts[1]);
                 return;
@@ -944,6 +948,156 @@ ${colors.bold('Details:')}`);
         }
         console.log('');
     }
+    async runSeo(url, showAll = false, jsonOutput = false) {
+        if (!url) {
+            url = this.currentDocUrl || this.baseUrl || '';
+            if (!url) {
+                console.log(colors.yellow('Usage: seo <url> [-a] [--format json]'));
+                console.log(colors.gray('  Examples: seo google.com | seo https://example.com -a'));
+                console.log(colors.gray('  -a, --all      Show all checks (including passed)'));
+                console.log(colors.gray('  --format json  Output raw JSON for programmatic use'));
+                console.log(colors.gray('  Or set a base URL first: url https://example.com'));
+                return;
+            }
+        }
+        else if (!url.startsWith('http') && !url.startsWith('-')) {
+            url = `https://${url}`;
+        }
+        if (!jsonOutput) {
+            console.log(colors.gray(`Analyzing SEO for ${url}...`));
+        }
+        const startTime = performance.now();
+        try {
+            const res = await this.client.get(url);
+            const html = await res.text();
+            const duration = Math.round(performance.now() - startTime);
+            const report = await analyzeSeo(html, { baseUrl: url });
+            if (jsonOutput) {
+                const jsonResult = {
+                    url,
+                    analyzedAt: new Date().toISOString(),
+                    durationMs: duration,
+                    score: report.score,
+                    grade: report.grade,
+                    title: report.title,
+                    metaDescription: report.metaDescription,
+                    content: report.content,
+                    headings: report.headings,
+                    links: report.links,
+                    images: report.images,
+                    openGraph: report.social.openGraph,
+                    twitterCard: report.social.twitterCard,
+                    jsonLd: report.jsonLd,
+                    technical: report.technical,
+                    checks: report.checks,
+                    summary: {
+                        total: report.checks.length,
+                        passed: report.checks.filter(c => c.status === 'pass').length,
+                        warnings: report.checks.filter(c => c.status === 'warn').length,
+                        errors: report.checks.filter(c => c.status === 'fail').length,
+                        info: report.checks.filter(c => c.status === 'info').length,
+                    },
+                };
+                console.log(JSON.stringify(jsonResult, null, 2));
+                this.lastResponse = jsonResult;
+                return;
+            }
+            let gradeColor = colors.red;
+            if (report.grade === 'A')
+                gradeColor = colors.green;
+            else if (report.grade === 'B')
+                gradeColor = colors.blue;
+            else if (report.grade === 'C')
+                gradeColor = colors.yellow;
+            else if (report.grade === 'D')
+                gradeColor = colors.magenta;
+            console.log(`
+${colors.bold(colors.cyan('🔍 SEO Analysis Report'))} ${colors.gray(`(${duration}ms)`)}
+Grade: ${gradeColor(colors.bold(report.grade))}  (${report.score}/100)
+`);
+            if (report.title) {
+                console.log(colors.bold('Title:') + ` ${report.title.text} ` + colors.gray(`(${report.title.length} chars)`));
+            }
+            if (report.metaDescription) {
+                const desc = report.metaDescription.text.length > 80
+                    ? report.metaDescription.text.slice(0, 77) + '...'
+                    : report.metaDescription.text;
+                console.log(colors.bold('Description:') + ` ${desc} ` + colors.gray(`(${report.metaDescription.length} chars)`));
+            }
+            if (report.content) {
+                console.log(colors.bold('Content:') + ` ${report.content.wordCount} words, ${report.content.paragraphCount} paragraphs, ~${report.content.readingTimeMinutes} min read`);
+            }
+            console.log('');
+            console.log(colors.bold('Checks:'));
+            const checksToShow = showAll
+                ? report.checks
+                : report.checks.filter(c => c.status !== 'pass');
+            const failed = checksToShow.filter(c => c.status === 'fail');
+            const warnings = checksToShow.filter(c => c.status === 'warn');
+            const info = checksToShow.filter(c => c.status === 'info');
+            const passed = showAll ? checksToShow.filter(c => c.status === 'pass') : [];
+            const displayCheck = (check) => {
+                let icon;
+                let nameColor;
+                switch (check.status) {
+                    case 'pass':
+                        icon = colors.green('✔');
+                        nameColor = colors.green;
+                        break;
+                    case 'warn':
+                        icon = colors.yellow('⚠');
+                        nameColor = colors.yellow;
+                        break;
+                    case 'fail':
+                        icon = colors.red('✖');
+                        nameColor = colors.red;
+                        break;
+                    default:
+                        icon = colors.blue('ℹ');
+                        nameColor = colors.blue;
+                }
+                console.log(`  ${icon} ${nameColor(check.name.padEnd(22))} ${check.message}`);
+                if (check.recommendation && check.status !== 'pass') {
+                    console.log(`     ${colors.gray('→')} ${colors.gray(check.recommendation)}`);
+                }
+                const evidence = check.evidence;
+                if (evidence && check.status !== 'pass') {
+                    if (evidence.found && Array.isArray(evidence.found) && evidence.found.length > 0) {
+                        const items = evidence.found.slice(0, 3);
+                        console.log(`      ${colors.gray('Found:')} ${colors.red(items.join(', '))}${evidence.found.length > 3 ? ` (+${evidence.found.length - 3} more)` : ''}`);
+                    }
+                    if (evidence.example) {
+                        console.log(`      ${colors.gray('Example:')} ${colors.cyan(evidence.example.split('\n')[0])}`);
+                    }
+                }
+            };
+            if (failed.length > 0) {
+                console.log(colors.red(`\n  Errors (${failed.length}):`));
+                failed.forEach(displayCheck);
+            }
+            if (warnings.length > 0) {
+                console.log(colors.yellow(`\n  Warnings (${warnings.length}):`));
+                warnings.forEach(displayCheck);
+            }
+            if (info.length > 0) {
+                console.log(colors.blue(`\n  Info (${info.length}):`));
+                info.forEach(displayCheck);
+            }
+            if (passed.length > 0) {
+                console.log(colors.green(`\n  Passed (${passed.length}):`));
+                passed.forEach(displayCheck);
+            }
+            if (!showAll && report.checks.filter(c => c.status === 'pass').length > 0) {
+                console.log(colors.gray(`\n  ${report.checks.filter(c => c.status === 'pass').length} checks passed. Use -a to show all.`));
+            }
+            console.log('');
+            this.lastResponse = report;
+        }
+        catch (error) {
+            console.error(colors.red(`SEO analysis failed: ${error.message}`));
+        }
+        console.log('');
+    }
     async runIpIntelligence(address) {
         if (!address) {
             console.log(colors.yellow('Usage: ip <address>'));
@@ -2182,6 +2336,9 @@ ${colors.bold('Network:')}
     ${colors.green('dns <domain>')}        Full DNS lookup (A, AAAA, MX, NS, SPF, DMARC).
     ${colors.green('rdap <domain>')}       RDAP lookup (modern WHOIS).
     ${colors.green('ping <host>')}         Quick TCP connectivity check.
+    ${colors.green('seo <url> [-a] [--format json]')} SEO analysis (70+ rules).
+                             ${colors.gray('-a, --all      Show all checks including passed')}
+                             ${colors.gray('--format json  Output raw JSON for programmatic use')}
 
   ${colors.bold('Web Scraping:')}
     ${colors.green('scrap <url>')}         Fetch and parse HTML document.

package/dist/index.d.ts

@@ -42,6 +42,7 @@ export * from './plugins/graphql.js';
 export * from './plugins/xml.js';
 export * from './plugins/scrape.js';
 export * from './scrape/index.js';
+export * from './seo/index.js';
 export * from './plugins/server-timing.js';
 export * from './plugins/auth.js';
 export * from './plugins/proxy-rotator.js';

package/dist/index.js

@@ -42,6 +42,7 @@ export * from './plugins/graphql.js';
 export * from './plugins/xml.js';
 export * from './plugins/scrape.js';
 export * from './scrape/index.js';
+export * from './seo/index.js';
 export * from './plugins/server-timing.js';
 export * from './plugins/auth.js';
 export * from './plugins/proxy-rotator.js';

package/dist/scrape/extractors.js

@@ -76,6 +76,7 @@ export function extractImages($, options) {
             height: height ? parseInt(height, 10) : undefined,
             srcset: $el.attr('srcset'),
             loading: $el.attr('loading'),
+            decoding: $el.attr('decoding'),
         });
     });
     return images;
@@ -117,7 +118,7 @@ export function extractMeta($) {
                 meta.author = content;
                 break;
             case 'robots':
-                meta.robots = content;
+                meta.robots = content.split(',').map((r) => r.trim().toLowerCase());
                 break;
             case 'viewport':
                 meta.viewport = content;

package/dist/scrape/types.d.ts

@@ -14,13 +14,14 @@ export interface ExtractedImage {
     height?: number;
     srcset?: string;
     loading?: 'lazy' | 'eager';
+    decoding?: 'async' | 'auto' | 'sync';
 }
 export interface ExtractedMeta {
     title?: string;
     description?: string;
     keywords?: string[];
     author?: string;
-    robots?: string;
+    robots?: string[];
     canonical?: string;
     viewport?: string;
     charset?: string;

package/dist/seo/analyzer.d.ts

@@ -0,0 +1,42 @@
+import type { CheerioAPI } from 'cheerio';
+import type { SeoReport, SeoAnalyzerOptions } from './types.js';
+import { type RulesEngineOptions } from './rules/index.js';
+export interface SeoAnalyzerFullOptions extends SeoAnalyzerOptions {
+    rules?: RulesEngineOptions;
+}
+export declare class SeoAnalyzer {
+    private $;
+    private options;
+    private rulesEngine;
+    constructor($: CheerioAPI, options?: SeoAnalyzerFullOptions);
+    static fromHtml(html: string, options?: SeoAnalyzerFullOptions): Promise<SeoAnalyzer>;
+    analyze(): SeoReport;
+    private buildRuleContext;
+    private analyzeUrlQuality;
+    private analyzeJsRendering;
+    private checkMixedContent;
+    private analyzeLinkSecurity;
+    private detectFavicon;
+    private analyzePerformanceHints;
+    private analyzeCWVHints;
+    private analyzeAccessibility;
+    private analyzeStructuralHtml;
+    private analyzeBreadcrumbs;
+    private analyzeMultimedia;
+    private analyzeTrustSignals;
+    private calculateTextHtmlRatio;
+    private convertToCheckResults;
+    private analyzeHeadings;
+    private analyzeContent;
+    private buildLinkAnalysis;
+    private buildImageAnalysis;
+    private buildSocialAnalysis;
+    private buildTechnicalAnalysis;
+    private calculateScore;
+    getRules(): import("./rules/types.js").SeoRule[];
+    getRulesByCategory(category: string): import("./rules/types.js").SeoRule[];
+    getCategories(): import("./rules/types.js").RuleCategory[];
+}
+export declare function analyzeSeo(html: string, options?: SeoAnalyzerFullOptions): Promise<SeoReport>;
+export { SEO_THRESHOLDS, createRulesEngine, SeoRulesEngine } from './rules/index.js';
+export type { RuleContext, RuleResult, RulesEngineOptions, RuleCategory, RuleSeverity, SeoRule } from './rules/index.js';