webpeel 0.20.2 → 0.20.3

package/dist/server/pg-job-queue.js

@@ -0,0 +1,375 @@
+/**
+ * PostgreSQL-backed job queue for production deployments
+ * Uses same Pool pattern as pg-auth-store.ts
+ */
+import pg from 'pg';
+import { randomUUID } from 'crypto';
+const { Pool } = pg;
+export class PostgresJobQueue {
+    pool;
+    cleanupInterval;
+    constructor(connectionString) {
+        const dbUrl = connectionString || process.env.DATABASE_URL;
+        if (!dbUrl) {
+            throw new Error('DATABASE_URL environment variable is required for PostgresJobQueue');
+        }
+        this.pool = new Pool({
+            connectionString: dbUrl,
+            // TLS: enabled when DATABASE_URL contains sslmode=require.
+            // Secure by default (rejectUnauthorized: true); set PG_REJECT_UNAUTHORIZED=false
+            // only for managed DBs (Render/Neon/Supabase) that use self-signed certs.
+            ssl: process.env.DATABASE_URL?.includes('sslmode=require')
+                ? { rejectUnauthorized: process.env.PG_REJECT_UNAUTHORIZED !== 'false' }
+                : undefined,
+            max: 20,
+            idleTimeoutMillis: 30000,
+            connectionTimeoutMillis: 10000,
+        });
+        // Initialize table
+        this.initTable().catch(err => {
+            console.error('Failed to initialize jobs table:', err);
+        });
+        // Clean up old completed/failed jobs every hour
+        this.cleanupInterval = setInterval(() => {
+            this.cleanupOldJobs().catch(err => {
+                console.error('Failed to cleanup old jobs:', err);
+            });
+        }, 60 * 60 * 1000);
+    }
+    /**
+     * Create jobs table if it doesn't exist
+     */
+    async initTable() {
+        try {
+            await this.pool.query(`
+        CREATE TABLE IF NOT EXISTS jobs (
+          id TEXT PRIMARY KEY,
+          type TEXT NOT NULL,
+          status TEXT NOT NULL DEFAULT 'pending',
+          progress INTEGER DEFAULT 0,
+          data JSONB,
+          error TEXT,
+          total INTEGER DEFAULT 0,
+          completed INTEGER DEFAULT 0,
+          credits_used INTEGER DEFAULT 0,
+          owner_id TEXT,
+          webhook_url TEXT,
+          webhook_events JSONB,
+          webhook_metadata JSONB,
+          webhook_secret TEXT,
+          created_at TIMESTAMPTZ DEFAULT NOW(),
+          updated_at TIMESTAMPTZ DEFAULT NOW(),
+          expires_at TIMESTAMPTZ
+        )
+      `);
+            // Add owner_id column if it doesn't exist (migration for existing tables)
+            await this.pool.query(`
+        DO $$ BEGIN
+          ALTER TABLE jobs ADD COLUMN IF NOT EXISTS owner_id TEXT;
+          ALTER TABLE jobs ADD COLUMN IF NOT EXISTS webhook_delivery JSONB;
+        EXCEPTION WHEN others THEN NULL;
+        END $$;
+      `);
+            // Add index on status and created_at for faster queries
+            await this.pool.query(`
+        CREATE INDEX IF NOT EXISTS idx_jobs_status_created 
+        ON jobs(status, created_at DESC)
+      `);
+            // Add index on type for filtering
+            await this.pool.query(`
+        CREATE INDEX IF NOT EXISTS idx_jobs_type 
+        ON jobs(type)
+      `);
+            // Add index on expires_at for cleanup
+            await this.pool.query(`
+        CREATE INDEX IF NOT EXISTS idx_jobs_expires 
+        ON jobs(expires_at)
+      `);
+            // Add index on owner_id for per-user job filtering
+            await this.pool.query(`
+        CREATE INDEX IF NOT EXISTS idx_jobs_owner 
+        ON jobs(owner_id)
+      `);
+        }
+        catch (error) {
+            console.error('Failed to create jobs table:', error);
+            throw error;
+        }
+    }
+    /**
+     * Create a new job
+     */
+    async createJob(type, webhook, ownerId) {
+        const id = randomUUID();
+        const now = new Date();
+        const expiresAt = new Date(now.getTime() + 25 * 60 * 60 * 1000); // 25h from now
+        try {
+            await this.pool.query(`INSERT INTO jobs (
+          id, type, status, progress, data, total, completed, credits_used,
+          owner_id, webhook_url, webhook_events, webhook_metadata, webhook_secret,
+          created_at, updated_at, expires_at
+        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)`, [
+                id,
+                type,
+                'queued',
+                0,
+                JSON.stringify([]),
+                0,
+                0,
+                0,
+                ownerId || null,
+                webhook?.url || null,
+                webhook?.events ? JSON.stringify(webhook.events) : null,
+                webhook?.metadata ? JSON.stringify(webhook.metadata) : null,
+                webhook?.secret || null,
+                now,
+                now,
+                expiresAt,
+            ]);
+            return {
+                id,
+                type,
+                status: 'queued',
+                progress: 0,
+                total: 0,
+                completed: 0,
+                creditsUsed: 0,
+                data: [],
+                webhook,
+                ownerId,
+                createdAt: now.toISOString(),
+                updatedAt: now.toISOString(),
+                expiresAt: expiresAt.toISOString(),
+            };
+        }
+        catch (error) {
+            console.error('Failed to create job:', error);
+            throw error;
+        }
+    }
+    /**
+     * Get a job by ID
+     */
+    async getJob(id) {
+        try {
+            const result = await this.pool.query(`SELECT * FROM jobs WHERE id = $1`, [id]);
+            if (result.rows.length === 0) {
+                return null;
+            }
+            return this.mapRowToJob(result.rows[0]);
+        }
+        catch (error) {
+            console.error('Failed to get job:', error);
+            return null;
+        }
+    }
+    /**
+     * Update a job
+     */
+    async updateJob(id, update) {
+        try {
+            const job = await this.getJob(id);
+            if (!job)
+                return;
+            const updates = [];
+            const values = [];
+            let paramIndex = 1;
+            // Map Job fields to database columns
+            if (update.status !== undefined) {
+                updates.push(`status = $${paramIndex++}`);
+                values.push(update.status);
+            }
+            if (update.progress !== undefined) {
+                updates.push(`progress = $${paramIndex++}`);
+                values.push(update.progress);
+            }
+            if (update.total !== undefined) {
+                updates.push(`total = $${paramIndex++}`);
+                values.push(update.total);
+            }
+            if (update.completed !== undefined) {
+                updates.push(`completed = $${paramIndex++}`);
+                values.push(update.completed);
+            }
+            if (update.creditsUsed !== undefined) {
+                updates.push(`credits_used = $${paramIndex++}`);
+                values.push(update.creditsUsed);
+            }
+            if (update.data !== undefined) {
+                updates.push(`data = $${paramIndex++}`);
+                values.push(JSON.stringify(update.data));
+            }
+            if (update.error !== undefined) {
+                updates.push(`error = $${paramIndex++}`);
+                values.push(update.error);
+            }
+            if (update.webhookDelivery !== undefined) {
+                updates.push(`webhook_delivery = $${paramIndex++}`);
+                values.push(JSON.stringify(update.webhookDelivery));
+            }
+            // Always update updated_at
+            updates.push(`updated_at = $${paramIndex++}`);
+            values.push(new Date());
+            // When job completes/fails, set expiration to 24h from now
+            if (update.status === 'completed' || update.status === 'failed' || update.status === 'cancelled') {
+                const expiresAt = new Date(Date.now() + 24 * 60 * 60 * 1000);
+                updates.push(`expires_at = $${paramIndex++}`);
+                values.push(expiresAt);
+            }
+            // Calculate progress percentage if total and completed are provided
+            const newTotal = update.total ?? job.total;
+            const newCompleted = update.completed ?? job.completed;
+            if (newTotal > 0) {
+                const progress = Math.round((newCompleted / newTotal) * 100);
+                if (!updates.some(u => u.startsWith('progress'))) {
+                    updates.push(`progress = $${paramIndex++}`);
+                    values.push(progress);
+                }
+            }
+            if (updates.length === 0)
+                return;
+            // Add job ID as the last parameter
+            values.push(id);
+            const sql = `UPDATE jobs SET ${updates.join(', ')} WHERE id = $${paramIndex}`;
+            await this.pool.query(sql, values);
+        }
+        catch (error) {
+            console.error('Failed to update job:', error);
+            throw error;
+        }
+    }
+    /**
+     * Cancel a job
+     */
+    async cancelJob(id) {
+        try {
+            const job = await this.getJob(id);
+            if (!job)
+                return false;
+            // Can only cancel queued or processing jobs
+            if (job.status !== 'queued' && job.status !== 'processing') {
+                return false;
+            }
+            await this.updateJob(id, {
+                status: 'cancelled',
+            });
+            return true;
+        }
+        catch (error) {
+            console.error('Failed to cancel job:', error);
+            return false;
+        }
+    }
+    /**
+     * List jobs with optional filters
+     */
+    async listJobs(options) {
+        try {
+            const conditions = [];
+            const values = [];
+            let paramIndex = 1;
+            if (options?.ownerId) {
+                conditions.push(`owner_id = $${paramIndex++}`);
+                values.push(options.ownerId);
+            }
+            if (options?.type) {
+                conditions.push(`type = $${paramIndex++}`);
+                values.push(options.type);
+            }
+            if (options?.status) {
+                conditions.push(`status = $${paramIndex++}`);
+                values.push(options.status);
+            }
+            const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+            const limit = options?.limit || 50;
+            const sql = `
+        SELECT * FROM jobs 
+        ${whereClause}
+        ORDER BY created_at DESC 
+        LIMIT $${paramIndex}
+      `;
+            values.push(limit);
+            const result = await this.pool.query(sql, values);
+            return result.rows.map(row => this.mapRowToJob(row));
+        }
+        catch (error) {
+            console.error('Failed to list jobs:', error);
+            return [];
+        }
+    }
+    /**
+     * Remove expired jobs (called periodically)
+     */
+    async cleanExpired() {
+        try {
+            await this.pool.query(`DELETE FROM jobs WHERE expires_at < NOW()`);
+        }
+        catch (error) {
+            console.error('Failed to clean expired jobs:', error);
+        }
+    }
+    /**
+     * Remove old completed/failed jobs (>7 days)
+     */
+    async cleanupOldJobs() {
+        try {
+            // Remove expired jobs
+            await this.cleanExpired();
+            // Remove completed/failed jobs older than 7 days
+            await this.pool.query(`DELETE FROM jobs 
+        WHERE (status = 'completed' OR status = 'failed' OR status = 'cancelled')
+        AND updated_at < NOW() - INTERVAL '7 days'`);
+        }
+        catch (error) {
+            console.error('Failed to cleanup old jobs:', error);
+        }
+    }
+    /**
+     * Map database row to Job object
+     */
+    mapRowToJob(row) {
+        const webhook = row.webhook_url
+            ? {
+                url: row.webhook_url,
+                events: row.webhook_events || [],
+                metadata: row.webhook_metadata || undefined,
+                secret: row.webhook_secret || undefined,
+            }
+            : undefined;
+        const webhookDelivery = row.webhook_delivery
+            ? (typeof row.webhook_delivery === 'string'
+                ? JSON.parse(row.webhook_delivery)
+                : row.webhook_delivery)
+            : undefined;
+        return {
+            id: row.id,
+            type: row.type,
+            status: row.status,
+            progress: row.progress || 0,
+            total: row.total || 0,
+            completed: row.completed || 0,
+            creditsUsed: row.credits_used || 0,
+            data: row.data || [],
+            error: row.error || undefined,
+            webhook,
+            webhookDelivery,
+            ownerId: row.owner_id || undefined,
+            createdAt: row.created_at.toISOString(),
+            updatedAt: row.updated_at.toISOString(),
+            expiresAt: row.expires_at.toISOString(),
+        };
+    }
+    /**
+     * Clean up interval on shutdown
+     */
+    destroy() {
+        clearInterval(this.cleanupInterval);
+    }
+    /**
+     * Close the database pool
+     */
+    async close() {
+        this.destroy();
+        await this.pool.end();
+    }
+}

package/dist/server/premium/domain-intel.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Domain Intelligence — premium server-only optimisation.
+ *
+ * Learns from historical fetch outcomes which domains require browser or
+ * stealth mode, so subsequent requests skip the slow simple→browser
+ * escalation path and go straight to the right strategy.
+ *
+ * Uses an exponential moving average for latency tracking and requires a
+ * minimum sample count before issuing recommendations to avoid false
+ * positives from one-off failures.
+ *
+ * This module is NOT shipped in the npm package.
+ */
+import type { StrategyHooks } from '../../core/strategy-hooks.js';
+export declare function clearDomainIntel(): void;
+export declare function createDomainIntelHooks(): Pick<StrategyHooks, 'getDomainRecommendation' | 'recordDomainResult'>;

package/dist/server/premium/domain-intel.js

@@ -0,0 +1,133 @@
+/**
+ * Domain Intelligence — premium server-only optimisation.
+ *
+ * Learns from historical fetch outcomes which domains require browser or
+ * stealth mode, so subsequent requests skip the slow simple→browser
+ * escalation path and go straight to the right strategy.
+ *
+ * Uses an exponential moving average for latency tracking and requires a
+ * minimum sample count before issuing recommendations to avoid false
+ * positives from one-off failures.
+ *
+ * This module is NOT shipped in the npm package.
+ */
+/* ---------- configuration ----------------------------------------------- */
+const MAX_DOMAINS = 500;
+const TTL_MS = 60 * 60 * 1000; // 1 hour
+const EMA_ALPHA = 0.3;
+const MIN_SAMPLES = 3;
+/* ---------- state ------------------------------------------------------- */
+const domainIntel = new Map();
+const methodCounts = new Map();
+/* ---------- internals --------------------------------------------------- */
+function domainKey(url) {
+    try {
+        return new URL(url).hostname.toLowerCase();
+    }
+    catch {
+        return '';
+    }
+}
+function prune(now) {
+    for (const [key, intel] of domainIntel) {
+        if (now - intel.lastSeen > TTL_MS) {
+            domainIntel.delete(key);
+            methodCounts.delete(key);
+        }
+    }
+}
+/* ---------- hook implementations ---------------------------------------- */
+function getDomainRecommendation(url) {
+    const key = domainKey(url);
+    if (!key)
+        return null;
+    const intel = domainIntel.get(key);
+    if (!intel)
+        return null;
+    const now = Date.now();
+    if (now - intel.lastSeen > TTL_MS) {
+        domainIntel.delete(key);
+        methodCounts.delete(key);
+        return null;
+    }
+    if (intel.sampleCount < MIN_SAMPLES)
+        return null;
+    const counts = methodCounts.get(key);
+    if (!counts)
+        return null;
+    // LRU touch
+    domainIntel.delete(key);
+    domainIntel.set(key, intel);
+    // All samples needed stealth → recommend stealth
+    if (counts.stealth === intel.sampleCount && intel.needsStealth) {
+        return { mode: 'stealth' };
+    }
+    // All samples needed browser (never succeeded with simple) → recommend browser
+    if (counts.simple === 0 &&
+        counts.browser + counts.stealth === intel.sampleCount &&
+        intel.needsBrowser) {
+        return { mode: 'browser' };
+    }
+    return null;
+}
+function recordDomainResult(url, method, latencyMs) {
+    const key = domainKey(url);
+    if (!key)
+        return;
+    const now = Date.now();
+    prune(now);
+    const existing = domainIntel.get(key);
+    const sanitizedLatency = Number.isFinite(latencyMs) && latencyMs > 0
+        ? latencyMs
+        : (existing?.avgLatencyMs ?? 0);
+    const next = existing
+        ? {
+            needsBrowser: existing.needsBrowser ||
+                method === 'browser' ||
+                method === 'stealth',
+            needsStealth: existing.needsStealth || method === 'stealth',
+            avgLatencyMs: existing.avgLatencyMs === 0
+                ? sanitizedLatency
+                : existing.avgLatencyMs * (1 - EMA_ALPHA) +
+                    sanitizedLatency * EMA_ALPHA,
+            lastSeen: now,
+            sampleCount: existing.sampleCount + 1,
+        }
+        : {
+            needsBrowser: method === 'browser' || method === 'stealth',
+            needsStealth: method === 'stealth',
+            avgLatencyMs: sanitizedLatency,
+            lastSeen: now,
+            sampleCount: 1,
+        };
+    const existingCounts = methodCounts.get(key) ?? {
+        simple: 0,
+        browser: 0,
+        stealth: 0,
+    };
+    existingCounts[method] += 1;
+    // Delete-then-set for LRU ordering
+    domainIntel.delete(key);
+    domainIntel.set(key, next);
+    methodCounts.set(key, existingCounts);
+    // Evict oldest when over capacity
+    while (domainIntel.size > MAX_DOMAINS) {
+        const oldest = domainIntel.keys().next().value;
+        if (!oldest)
+            break;
+        domainIntel.delete(oldest);
+        methodCounts.delete(oldest);
+    }
+}
+/* ---------- cleanup ----------------------------------------------------- */
+export function clearDomainIntel() {
+    domainIntel.clear();
+    methodCounts.clear();
+}
+/* ---------- public export ----------------------------------------------- */
+export function createDomainIntelHooks() {
+    return {
+        getDomainRecommendation,
+        recordDomainResult,
+    };
+}

package/dist/server/premium/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Premium strategy hooks — server-only optimisations.
+ *
+ * Call `registerPremiumHooks()` once at server startup to activate:
+ *   • SWR (stale-while-revalidate) response cache
+ *   • Domain intelligence (learns which sites need browser/stealth)
+ *   • Parallel race strategy (starts browser if simple fetch is slow)
+ *
+ * These modules are NOT shipped in the npm package.
+ */
+export { clearDomainIntel } from './domain-intel.js';
+/**
+ * Wire all premium hooks into the core strategy layer.
+ *
+ * Must be called before any request is served.
+ */
+export declare function registerPremiumHooks(): void;

package/dist/server/premium/index.js

@@ -0,0 +1,35 @@
+/**
+ * Premium strategy hooks — server-only optimisations.
+ *
+ * Call `registerPremiumHooks()` once at server startup to activate:
+ *   • SWR (stale-while-revalidate) response cache
+ *   • Domain intelligence (learns which sites need browser/stealth)
+ *   • Parallel race strategy (starts browser if simple fetch is slow)
+ *
+ * These modules are NOT shipped in the npm package.
+ */
+import { registerStrategyHooks } from '../../core/strategy-hooks.js';
+import { createSWRCacheHooks } from './swr-cache.js';
+import { createDomainIntelHooks } from './domain-intel.js';
+export { clearDomainIntel } from './domain-intel.js';
+/**
+ * Wire all premium hooks into the core strategy layer.
+ *
+ * Must be called before any request is served.
+ */
+export function registerPremiumHooks() {
+    const cacheHooks = createSWRCacheHooks();
+    const intelHooks = createDomainIntelHooks();
+    registerStrategyHooks({
+        // SWR cache
+        checkCache: cacheHooks.checkCache,
+        markRevalidating: cacheHooks.markRevalidating,
+        setCache: cacheHooks.setCache,
+        // Domain intelligence
+        getDomainRecommendation: intelHooks.getDomainRecommendation,
+        recordDomainResult: intelHooks.recordDomainResult,
+        // Parallel race strategy
+        shouldRace: () => true,
+        getRaceTimeoutMs: () => 2000,
+    });
+}

package/dist/server/premium/swr-cache.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Stale-While-Revalidate cache — premium server-only optimisation.
+ *
+ * Wraps the core LRU cache with SWR semantics:
+ *   • Fresh entries are served immediately.
+ *   • Stale entries (within the SWR window) are served AND trigger a
+ *     background revalidation so the next caller gets a fresh result.
+ *   • Expired entries (past the SWR window) are evicted.
+ *
+ * This module is NOT shipped in the npm package — it lives under
+ * `src/server/` which is excluded from the package.json `files` list.
+ */
+import type { StrategyHooks } from '../../core/strategy-hooks.js';
+export declare function createSWRCacheHooks(): Pick<StrategyHooks, 'checkCache' | 'markRevalidating' | 'setCache'>;

package/dist/server/premium/swr-cache.js

@@ -0,0 +1,34 @@
+/**
+ * Stale-While-Revalidate cache — premium server-only optimisation.
+ *
+ * Wraps the core LRU cache with SWR semantics:
+ *   • Fresh entries are served immediately.
+ *   • Stale entries (within the SWR window) are served AND trigger a
+ *     background revalidation so the next caller gets a fresh result.
+ *   • Expired entries (past the SWR window) are evicted.
+ *
+ * This module is NOT shipped in the npm package — it lives under
+ * `src/server/` which is excluded from the package.json `files` list.
+ */
+import { getCachedWithSWR, markRevalidating, setCached, } from '../../core/cache.js';
+/* ---------- hook implementations ---------------------------------------- */
+function checkCache(url) {
+    const entry = getCachedWithSWR(url);
+    if (!entry)
+        return null;
+    return { value: entry.value, stale: entry.stale };
+}
+function markRevalidatingHook(url) {
+    return markRevalidating(url);
+}
+function setCache(url, result) {
+    setCached(url, result);
+}
+/* ---------- public export ----------------------------------------------- */
+export function createSWRCacheHooks() {
+    return {
+        checkCache,
+        markRevalidating: markRevalidatingHook,
+        setCache,
+    };
+}

package/dist/server/routes/activity.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Activity endpoint - provides recent API request history
+ */
+import { Router } from 'express';
+import { AuthStore } from '../auth-store.js';
+export declare function createActivityRouter(authStore: AuthStore): Router;