webpeel 0.20.2 → 0.20.3

package/dist/server/middleware/rate-limit.js

@@ -0,0 +1,167 @@
+/**
+ * Sliding window rate limiting middleware
+ */
+import '../types.js'; // Augments Express.Request with requestId
+export class RateLimiter {
+    store = new Map();
+    windowMs;
+    constructor(windowMs = 60000) {
+        this.windowMs = windowMs;
+    }
+    /**
+     * Check if request is allowed under rate limit
+     * @param cost - Number of credits this request costs (default: 1)
+     */
+    checkLimit(identifier, limit, cost = 1) {
+        const now = Date.now();
+        const windowStart = now - this.windowMs;
+        // Get or create entry
+        let entry = this.store.get(identifier);
+        if (!entry) {
+            entry = { timestamps: [] };
+            this.store.set(identifier, entry);
+        }
+        // Remove timestamps outside the window
+        entry.timestamps = entry.timestamps.filter(ts => ts > windowStart);
+        // Check if limit would be exceeded by this request's cost
+        if (entry.timestamps.length + cost > limit) {
+            const oldestTimestamp = entry.timestamps[0];
+            const retryAfter = oldestTimestamp
+                ? Math.ceil((oldestTimestamp + this.windowMs - now) / 1000)
+                : 1;
+            return {
+                allowed: false,
+                remaining: Math.max(0, limit - entry.timestamps.length),
+                retryAfter,
+            };
+        }
+        // Add `cost` timestamps to represent the weight of this request
+        for (let i = 0; i < cost; i++) {
+            entry.timestamps.push(now);
+        }
+        return {
+            allowed: true,
+            remaining: limit - entry.timestamps.length,
+        };
+    }
+    /**
+     * Clean up old entries (call periodically)
+     */
+    cleanup() {
+        const now = Date.now();
+        const windowStart = now - this.windowMs;
+        for (const [identifier, entry] of this.store.entries()) {
+            entry.timestamps = entry.timestamps.filter(ts => ts > windowStart);
+            if (entry.timestamps.length === 0) {
+                this.store.delete(identifier);
+            }
+        }
+    }
+}
+/**
+ * Hourly burst limits per tier.
+ * These are the hard caps enforced by the in-memory sliding window.
+ * Free: 50/hr, Pro: 100/hr, Max: 500/hr (matches pricing page + stripe.ts).
+ */
+const TIER_BURST_LIMITS = {
+    free: 50,
+    pro: 100,
+    max: 500,
+    admin: 999999,
+};
+/**
+ * System/documentation endpoints that should be exempt from rate limiting.
+ * These are low-cost informational endpoints that should never be throttled.
+ */
+const EXEMPT_PATHS = [
+    '/health',
+    '/openapi.json',
+    '/openapi.yaml',
+    '/docs',
+    '/v1/usage',
+    '/v1/me',
+    '/v1/keys',
+    '/v1/activity',
+    '/v1/stats',
+];
+export function createRateLimitMiddleware(limiter) {
+    return (req, res, next) => {
+        try {
+            // Skip rate limiting for system/documentation endpoints
+            if (EXEMPT_PATHS.some(p => req.path === p || req.path.startsWith(p + '/'))) {
+                return next();
+            }
+            // Use API key or real client IP as identifier.
+            // Prefer Cloudflare CF-Connecting-IP, then x-forwarded-for first
+            // entry (real client), then x-real-ip, then req.ip.
+            const forwardedFor = req.headers['x-forwarded-for'];
+            const firstForwardedIp = typeof forwardedFor === 'string'
+                ? forwardedFor.split(',')[0].trim()
+                : Array.isArray(forwardedFor) ? forwardedFor[0] : undefined;
+            const clientIp = req.headers['cf-connecting-ip']
+                || firstForwardedIp
+                || req.headers['x-real-ip']
+                || req.ip
+                || 'unknown';
+            const identifier = req.auth?.keyInfo?.key || clientIp;
+            // Use tier-based hourly burst limits (matches the 1-hour sliding window)
+            const limit = TIER_BURST_LIMITS[req.auth?.tier || 'free'] || 50;
+            // Weighted cost based on route — heavier operations consume more credits
+            let cost = 1;
+            const path = req.path;
+            if (path.includes('/crawl') || path.includes('/map'))
+                cost = 5;
+            else if (path.includes('/batch'))
+                cost = 2;
+            else if (path.includes('/screenshot'))
+                cost = 2;
+            else if (req.query.render === 'true' || req.body?.render === true)
+                cost = 3;
+            const result = limiter.checkLimit(identifier, limit, cost);
+            // Calculate reset timestamp
+            const now = Date.now();
+            const resetTimestamp = Math.ceil((now + limiter['windowMs']) / 1000);
+            // Set rate limit headers on ALL responses
+            res.setHeader('X-RateLimit-Limit', limit.toString());
+            res.setHeader('X-RateLimit-Remaining', Math.max(0, result.remaining).toString());
+            res.setHeader('X-RateLimit-Reset', resetTimestamp.toString());
+            // Add plan header if authenticated
+            if (req.auth?.tier) {
+                res.setHeader('X-WebPeel-Plan', req.auth.tier);
+            }
+            if (!result.allowed) {
+                const retryAfterSecs = result.retryAfter;
+                res.setHeader('Retry-After', retryAfterSecs.toString());
+                const tier = req.auth?.tier || 'free';
+                const upgradeHint = tier === 'free'
+                    ? ' Upgrade to Pro ($9/mo) for 100/hr burst limit → https://webpeel.dev/pricing'
+                    : tier === 'pro'
+                        ? ' Upgrade to Max ($29/mo) for 500/hr burst limit → https://webpeel.dev/pricing'
+                        : '';
+                res.status(429).json({
+                    success: false,
+                    error: {
+                        type: 'rate_limited',
+                        message: `Hourly rate limit exceeded (${limit} requests/hr on ${tier} plan). Try again in ${retryAfterSecs}s.`,
+                        hint: `Retry after ${retryAfterSecs} seconds.${upgradeHint}`,
+                        docs: 'https://webpeel.dev/docs/errors#rate-limited',
+                    },
+                    metadata: { requestId: req.requestId },
+                });
+                return; // Stop processing - rate limit exceeded
+            }
+            next();
+        }
+        catch (_error) {
+            res.status(500).json({
+                success: false,
+                error: {
+                    type: 'internal_error',
+                    message: 'Rate limiting failed',
+                    docs: 'https://webpeel.dev/docs/errors#internal-error',
+                },
+                metadata: { requestId: req.requestId },
+            });
+        }
+    };
+}

package/dist/server/middleware/url-validator.d.ts

@@ -0,0 +1,15 @@
+/**
+ * URL validation middleware to prevent SSRF attacks
+ * Validates URLs BEFORE any network request is made
+ */
+/**
+ * Validate URL to prevent SSRF attacks
+ * Blocks localhost, private IPs, link-local addresses, and non-HTTP(S) protocols
+ */
+export declare function validateUrlForSSRF(urlString: string): void;
+/**
+ * SSRF Error class
+ */
+export declare class SSRFError extends Error {
+    constructor(message: string);
+}

package/dist/server/middleware/url-validator.js

@@ -0,0 +1,186 @@
+/**
+ * URL validation middleware to prevent SSRF attacks
+ * Validates URLs BEFORE any network request is made
+ */
+/**
+ * Validate URL to prevent SSRF attacks
+ * Blocks localhost, private IPs, link-local addresses, and non-HTTP(S) protocols
+ */
+export function validateUrlForSSRF(urlString) {
+    // Parse URL
+    let url;
+    try {
+        url = new URL(urlString);
+    }
+    catch {
+        throw new Error('Invalid URL format');
+    }
+    // Only allow HTTP(S)
+    if (!['http:', 'https:'].includes(url.protocol)) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    const hostname = url.hostname.toLowerCase();
+    // Block localhost patterns
+    const localhostPatterns = ['localhost', '0.0.0.0'];
+    if (localhostPatterns.some(pattern => hostname === pattern || hostname.endsWith('.' + pattern))) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Parse and validate IP addresses
+    const ipv4Info = parseIPv4(hostname);
+    if (ipv4Info) {
+        validateIPv4ForSSRF(ipv4Info);
+    }
+    // Validate IPv6
+    if (hostname.includes(':')) {
+        validateIPv6ForSSRF(hostname);
+    }
+}
+/**
+ * SSRF Error class
+ */
+export class SSRFError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'SSRFError';
+    }
+}
+/**
+ * Parse IPv4 address in any format (dotted, hex, octal, decimal)
+ */
+function parseIPv4(hostname) {
+    const cleaned = hostname.replace(/^\[|\]$/g, '');
+    // Standard dotted notation: 192.168.1.1
+    const dottedRegex = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/;
+    const dottedMatch = cleaned.match(dottedRegex);
+    if (dottedMatch) {
+        const octets = dottedMatch.slice(1).map(Number);
+        if (octets.every(o => o >= 0 && o <= 255)) {
+            return octets;
+        }
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Hex notation: 0x7f000001
+    if (/^0x[0-9a-fA-F]+$/.test(cleaned)) {
+        const num = parseInt(cleaned, 16);
+        return [
+            (num >>> 24) & 0xff,
+            (num >>> 16) & 0xff,
+            (num >>> 8) & 0xff,
+            num & 0xff,
+        ];
+    }
+    // Octal notation
+    if (/^0[0-7]/.test(cleaned)) {
+        if (/^0[0-7]+$/.test(cleaned)) {
+            const num = parseInt(cleaned, 8);
+            if (num <= 0xffffffff) {
+                return [
+                    (num >>> 24) & 0xff,
+                    (num >>> 16) & 0xff,
+                    (num >>> 8) & 0xff,
+                    num & 0xff,
+                ];
+            }
+        }
+        const parts = cleaned.split('.');
+        if (parts.length === 4) {
+            const octets = parts.map(p => parseInt(p, /^0[0-7]/.test(p) ? 8 : 10));
+            if (octets.every(o => o >= 0 && o <= 255)) {
+                return octets;
+            }
+        }
+    }
+    // Decimal notation: 2130706433
+    if (/^\d+$/.test(cleaned)) {
+        const num = parseInt(cleaned, 10);
+        if (num <= 0xffffffff) {
+            return [
+                (num >>> 24) & 0xff,
+                (num >>> 16) & 0xff,
+                (num >>> 8) & 0xff,
+                num & 0xff,
+            ];
+        }
+    }
+    return null;
+}
+/**
+ * Validate IPv4 address against private/reserved ranges
+ */
+function validateIPv4ForSSRF(octets) {
+    const [a, b, c, d] = octets;
+    // Loopback: 127.0.0.0/8
+    if (a === 127) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Private: 10.0.0.0/8
+    if (a === 10) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Private: 172.16.0.0/12
+    if (a === 172 && b >= 16 && b <= 31) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Private: 192.168.0.0/16
+    if (a === 192 && b === 168) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Link-local: 169.254.0.0/16 (includes AWS metadata endpoint)
+    if (a === 169 && b === 254) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Broadcast: 255.255.255.255
+    if (a === 255 && b === 255 && c === 255 && d === 255) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // This network: 0.0.0.0/8
+    if (a === 0) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+}
+/**
+ * Validate IPv6 address against private/reserved ranges
+ */
+function validateIPv6ForSSRF(hostname) {
+    const addr = hostname.replace(/^\[|\]$/g, '').toLowerCase();
+    // Loopback: ::1
+    if (addr === '::1' || addr === '0:0:0:0:0:0:0:1') {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // IPv6 mapped IPv4: ::ffff:192.168.1.1
+    if (addr.startsWith('::ffff:')) {
+        const ipv4Part = addr.substring(7);
+        if (ipv4Part.includes('.')) {
+            const parts = ipv4Part.split('.');
+            if (parts.length === 4) {
+                const octets = parts.map(p => parseInt(p, 10));
+                if (octets.every(o => !isNaN(o) && o >= 0 && o <= 255)) {
+                    validateIPv4ForSSRF(octets);
+                }
+            }
+        }
+        else {
+            const hexStr = ipv4Part.replace(/:/g, '');
+            if (/^[0-9a-f]{1,8}$/.test(hexStr)) {
+                const num = parseInt(hexStr, 16);
+                const octets = [
+                    (num >>> 24) & 0xff,
+                    (num >>> 16) & 0xff,
+                    (num >>> 8) & 0xff,
+                    num & 0xff,
+                ];
+                validateIPv4ForSSRF(octets);
+            }
+        }
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Unique local addresses: fc00::/7
+    if (addr.startsWith('fc') || addr.startsWith('fd')) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+    // Link-local: fe80::/10
+    if (addr.startsWith('fe8') || addr.startsWith('fe9') ||
+        addr.startsWith('fea') || addr.startsWith('feb')) {
+        throw new SSRFError('Cannot fetch localhost, private networks, or non-HTTP URLs');
+    }
+}