webpeel 0.21.53 → 0.21.57

package/dist/core/apply.js

@@ -590,7 +590,6 @@ export async function applyToJob(options) {
     try {
         // ── 3. Launch persistent browser ──────────────────────────────
         progress('navigating', 'Launching browser with persistent session...');
-        // @ts-expect-error rebrowser-playwright types diverge from upstream playwright-core
         context = await stealthChromium.launchPersistentContext(sessionDir, {
             headless: false, // visible so user can monitor (or log in on first run)
             viewport: { width: 1440, height: 900 },

package/dist/core/pipeline.js

@@ -1217,52 +1217,6 @@ export async function finalize(ctx) {
             log.error('Change tracking failed:', error);
         }
     }
-    // ── Auto-escalation: retry thin content with browser rendering ──────────────
-    // If simple fetch returned very little content and user didn't explicitly disable render,
-    // automatically retry with browser rendering to handle JS-heavy/paywalled sites.
-    const preEscalationWords = ctx.content.trim().split(/\s+/).filter((w) => w.length > 0).length;
-    const escalationFetchMethod = fetchResult?.method || 'unknown';
-    const alreadyTriedBrowser = escalationFetchMethod === 'browser' || escalationFetchMethod === 'stealth'
-        || options.render || options.stealth;
-    const userDisabledRender = options.render === false;
-    const escalationCandidate = preEscalationWords < 200 && preEscalationWords > 0
-        && escalationFetchMethod === 'simple' && !alreadyTriedBrowser && !userDisabledRender
-        && !ctx._escalated;
-    if (escalationCandidate) {
-        log.info(`thin content (${preEscalationWords}w) from simple fetch, auto-escalating to browser render for ${ctx.url}`);
-        ctx._escalated = true;
-        try {
-            const { smartFetch } = await import('./strategies.js');
-            const browserResult = await smartFetch(ctx.url, {
-                forceBrowser: true,
-                stealth: false,
-                timeoutMs: options.timeout || 15000,
-                proxy: options.proxy,
-            });
-            if (browserResult.html && browserResult.html.length > (fetchResult?.html?.length || 0)) {
-                const { htmlToMarkdown } = await import('./markdown.js');
-                const browserContent = htmlToMarkdown(browserResult.html);
-                const browserWords = browserContent.trim().split(/\s+/).filter((w) => w.length > 0).length;
-                if (browserWords > preEscalationWords) {
-                    log.info(`browser escalation improved content: ${preEscalationWords}w → ${browserWords}w`);
-                    ctx.content = browserContent;
-                    ctx.fetchResult = browserResult;
-                    ctx.fetchResult.method = 'browser-escalation';
-                }
-                else {
-                    log.debug(`browser escalation did not improve (${browserWords}w vs ${preEscalationWords}w)`);
-                }
-                // Always clean up browser resources
-                if (browserResult.page)
-                    await browserResult.page.close().catch(() => { });
-                if (browserResult.browser)
-                    await browserResult.browser.close().catch(() => { });
-            }
-        }
-        catch (e) {
-            log.debug('browser escalation failed:', e instanceof Error ? e.message : e);
-        }
-    }
     // Generate AI summary if requested
     if (options.summary && options.llm) {
         try {

package/dist/core/source-credibility.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Source credibility scoring — lightweight, zero dependencies.
+ *
+ * Classifies URLs by trustworthiness:
+ *   - Official (★★★): .gov, .edu, .mil, WHO, NIH, academic journals
+ *   - Verified (★★):  Wikipedia, Reuters, BBC, GitHub, StackOverflow
+ *   - General (★):    Everything else
+ */
+export interface SourceCredibility {
+    tier: 'official' | 'verified' | 'general';
+    stars: number;
+    label: string;
+}
+/**
+ * Assess the credibility of a source URL.
+ */
+export declare function getSourceCredibility(url: string): SourceCredibility;

package/dist/core/source-credibility.js

@@ -0,0 +1,83 @@
+/**
+ * Source credibility scoring — lightweight, zero dependencies.
+ *
+ * Classifies URLs by trustworthiness:
+ *   - Official (★★★): .gov, .edu, .mil, WHO, NIH, academic journals
+ *   - Verified (★★):  Wikipedia, Reuters, BBC, GitHub, StackOverflow
+ *   - General (★):    Everything else
+ */
+/** Official TLDs and hostnames that indicate high-authority sources */
+const OFFICIAL_TLDS = new Set(['.gov', '.edu', '.mil']);
+const OFFICIAL_HOSTNAMES = new Set([
+    // Academic / research
+    'arxiv.org', 'scholar.google.com', 'pubmed.ncbi.nlm.nih.gov', 'ncbi.nlm.nih.gov',
+    'jstor.org', 'nature.com', 'science.org', 'cell.com', 'nejm.org', 'bmj.com',
+    'thelancet.com', 'plos.org', 'springer.com', 'elsevier.com',
+    // International organisations
+    'who.int', 'un.org', 'worldbank.org', 'imf.org', 'oecd.org', 'europa.eu',
+    // Official tech documentation
+    'docs.python.org', 'developer.mozilla.org', 'nodejs.org', 'rust-lang.org',
+    'docs.microsoft.com', 'learn.microsoft.com', 'developer.apple.com',
+    'developer.android.com', 'php.net', 'ruby-lang.org', 'golang.org', 'go.dev',
+    // Health / medicine
+    'cdc.gov', 'nih.gov', 'fda.gov', 'mayoclinic.org', 'clevelandclinic.org',
+    'webmd.com', 'medlineplus.gov',
+    // Standards / specs
+    'w3.org', 'ietf.org', 'rfc-editor.org', 'iso.org',
+]);
+const VERIFIED_HOSTNAMES = new Set([
+    // Encyclopaedia / reference
+    'wikipedia.org', 'en.wikipedia.org', 'britannica.com',
+    // Reputable news agencies
+    'reuters.com', 'apnews.com', 'bbc.com', 'bbc.co.uk', 'nytimes.com',
+    'washingtonpost.com', 'theguardian.com', 'economist.com', 'ft.com',
+    'cnn.com', 'npr.org', 'pbs.org',
+    // Developer resources
+    'github.com', 'stackoverflow.com', 'npmjs.com', 'pypi.org',
+    'crates.io', 'docs.rs', 'packagist.org', 'rubygems.org',
+    // Official cloud / vendor docs
+    'docs.aws.amazon.com', 'cloud.google.com', 'docs.github.com',
+    'azure.microsoft.com', 'registry.terraform.io',
+    // Reputable tech publications
+    'arstechnica.com', 'wired.com', 'techcrunch.com', 'theverge.com',
+    // National Geographic, Smithsonian
+    'nationalgeographic.com', 'smithsonianmag.com',
+]);
+/**
+ * Assess the credibility of a source URL.
+ */
+export function getSourceCredibility(url) {
+    try {
+        const hostname = new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+        // Check official TLDs
+        for (const tld of OFFICIAL_TLDS) {
+            if (hostname.endsWith(tld)) {
+                return { tier: 'official', stars: 3, label: 'OFFICIAL SOURCE' };
+            }
+        }
+        // Check known official hostnames  
+        if (OFFICIAL_HOSTNAMES.has(hostname)) {
+            return { tier: 'official', stars: 3, label: 'OFFICIAL SOURCE' };
+        }
+        // Check parent domain (e.g. en.wikipedia.org → wikipedia.org)
+        const parts = hostname.split('.');
+        if (parts.length > 2) {
+            const parentDomain = parts.slice(-2).join('.');
+            if (OFFICIAL_HOSTNAMES.has(parentDomain)) {
+                return { tier: 'official', stars: 3, label: 'OFFICIAL SOURCE' };
+            }
+            if (VERIFIED_HOSTNAMES.has(parentDomain)) {
+                return { tier: 'verified', stars: 2, label: 'VERIFIED' };
+            }
+        }
+        // Check known verified hostnames
+        if (VERIFIED_HOSTNAMES.has(hostname)) {
+            return { tier: 'verified', stars: 2, label: 'VERIFIED' };
+        }
+        // Everything else
+        return { tier: 'general', stars: 1, label: 'UNVERIFIED' };
+    }
+    catch {
+        return { tier: 'general', stars: 1, label: 'UNVERIFIED' };
+    }
+}

package/dist/server/app.js

@@ -43,6 +43,7 @@ import { createPlaygroundRouter } from './routes/playground.js';
 import { createReaderRouter } from './routes/reader.js';
 import { createSharePublicRouter, createShareRouter } from './routes/share.js';
 import { createJobQueue } from './job-queue.js';
+import { createQueueFetchRouter } from './routes/fetch-queue.js';
 import { createCompatRouter } from './routes/compat.js';
 import { createCrawlRouter } from './routes/crawl.js';
 import { createMapRouter } from './routes/map.js';
@@ -229,7 +230,8 @@ export function createApp(config = {}) {
     }, 5 * 60 * 1000);
     // Health check MUST be before auth/rate-limit middleware
     // Render hits /health every ~30s; rate-limiting it causes 429 → service marked as failed
-    app.use(createHealthRouter());
+    // Pass pool so /ready can check DB connectivity
+    app.use(createHealthRouter(pool));
     // OpenAPI spec — public, no auth required
     app.get('/openapi.yaml', (_req, res) => {
         res.setHeader('Content-Type', 'application/yaml; charset=utf-8');
@@ -287,7 +289,15 @@ export function createApp(config = {}) {
         app.use(createWatchRouter(pool));
     }
     // /v1/fetch, /v1/search — all scopes allowed, no guard needed
-    app.use(createFetchRouter(authStore));
+    // In queue mode (API_MODE=queue), /v1/fetch and /v1/render are replaced by
+    // queue-backed endpoints that enqueue Bull jobs and return { jobId, status }.
+    // GET /v1/jobs/:id is also provided by the queue router for result polling.
+    if (process.env.API_MODE === 'queue') {
+        app.use(createQueueFetchRouter());
+    }
+    else {
+        app.use(createFetchRouter(authStore));
+    }
     // /v1/screenshot — full or read only (router uses absolute paths, guard before router)
     app.use('/v1/screenshot', requireScope('full', 'read'));
     app.use(createScreenshotRouter(authStore));

package/dist/server/bull-queues.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Bull queue setup for WebPeel microservices.
+ *
+ * Used by:
+ *   - API container  (API_MODE=queue)  — to enqueue fetch/render jobs
+ *   - Worker container (WORKER_MODE=1) — to process jobs and write results
+ *
+ * Queue names:
+ *   - "webpeel:fetch"  — HTTP-only fetches (no browser)
+ *   - "webpeel:render" — Browser/Playwright fetches (render=true)
+ *
+ * Job result format stored in Redis:
+ *   key: webpeel:result:<jobId>
+ *   value: JSON string of { status, result?, error? }
+ *   TTL: 24h
+ *
+ * No secrets in code. All config via env vars:
+ *   REDIS_URL      — e.g. redis://redis:6379 (default)
+ *   REDIS_PASSWORD — optional password
+ */
+import Bull from 'bull';
+export interface FetchJobPayload {
+    jobId: string;
+    url: string;
+    format?: 'markdown' | 'text' | 'html' | 'clean';
+    render?: boolean;
+    wait?: number;
+    maxTokens?: number;
+    budget?: number;
+    stealth?: boolean;
+    screenshot?: boolean;
+    fullPage?: boolean;
+    selector?: string;
+    exclude?: string[];
+    includeTags?: string[];
+    excludeTags?: string[];
+    images?: boolean;
+    actions?: any[];
+    timeout?: number;
+    lite?: boolean;
+    raw?: boolean;
+    readable?: boolean;
+    question?: string;
+    userId?: string;
+}
+export declare function getFetchQueue(): Bull.Queue<FetchJobPayload>;
+export declare function getRenderQueue(): Bull.Queue<FetchJobPayload>;
+export declare const RESULT_KEY_PREFIX = "webpeel:result:";
+export declare const RESULT_TTL_SECONDS = 86400;
+export interface JobResult {
+    status: 'queued' | 'processing' | 'completed' | 'failed';
+    result?: any;
+    error?: string;
+    startedAt?: string;
+    completedAt?: string;
+}
+export declare function closeQueues(): Promise<void>;

package/dist/server/bull-queues.js

@@ -0,0 +1,86 @@
+/**
+ * Bull queue setup for WebPeel microservices.
+ *
+ * Used by:
+ *   - API container  (API_MODE=queue)  — to enqueue fetch/render jobs
+ *   - Worker container (WORKER_MODE=1) — to process jobs and write results
+ *
+ * Queue names:
+ *   - "webpeel:fetch"  — HTTP-only fetches (no browser)
+ *   - "webpeel:render" — Browser/Playwright fetches (render=true)
+ *
+ * Job result format stored in Redis:
+ *   key: webpeel:result:<jobId>
+ *   value: JSON string of { status, result?, error? }
+ *   TTL: 24h
+ *
+ * No secrets in code. All config via env vars:
+ *   REDIS_URL      — e.g. redis://redis:6379 (default)
+ *   REDIS_PASSWORD — optional password
+ */
+import Bull from 'bull';
+// ─── Redis connection config ─────────────────────────────────────────────────
+function getRedisConfig() {
+    const url = process.env.REDIS_URL || 'redis://redis:6379';
+    const password = process.env.REDIS_PASSWORD || undefined;
+    // Parse the URL to extract host/port (Bull accepts host+port or full URL)
+    try {
+        const parsed = new URL(url);
+        return {
+            host: parsed.hostname,
+            port: parseInt(parsed.port || '6379', 10),
+            password,
+            db: parseInt(parsed.pathname?.slice(1) || '0', 10) || 0,
+        };
+    }
+    catch {
+        // Fallback defaults
+        return {
+            host: 'redis',
+            port: 6379,
+            password,
+        };
+    }
+}
+const sharedOpts = {
+    redis: getRedisConfig(),
+    defaultJobOptions: {
+        attempts: 3,
+        backoff: {
+            type: 'exponential',
+            delay: 2000,
+        },
+        removeOnComplete: false, // keep for result lookup
+        removeOnFail: false,
+        timeout: 120_000, // 2 min hard timeout per job
+    },
+};
+// ─── Queue singletons ────────────────────────────────────────────────────────
+let _fetchQueue = null;
+let _renderQueue = null;
+export function getFetchQueue() {
+    if (!_fetchQueue) {
+        _fetchQueue = new Bull('webpeel:fetch', sharedOpts);
+    }
+    return _fetchQueue;
+}
+export function getRenderQueue() {
+    if (!_renderQueue) {
+        _renderQueue = new Bull('webpeel:render', sharedOpts);
+    }
+    return _renderQueue;
+}
+// ─── Result helpers (Redis key = webpeel:result:<jobId>) ─────────────────────
+export const RESULT_KEY_PREFIX = 'webpeel:result:';
+export const RESULT_TTL_SECONDS = 86_400; // 24 hours
+// ─── Graceful teardown ───────────────────────────────────────────────────────
+export async function closeQueues() {
+    const closes = [];
+    if (_fetchQueue)
+        closes.push(_fetchQueue.close());
+    if (_renderQueue)
+        closes.push(_renderQueue.close());
+    await Promise.all(closes);
+    _fetchQueue = null;
+    _renderQueue = null;
+}

package/dist/server/middleware/auth.js

@@ -20,7 +20,7 @@ export function createAuthMiddleware(authStore) {
             const apiKeyHeader = req.headers['x-api-key'];
             // SECURITY: Skip API key auth for public/JWT-protected endpoints
             // These routes either need no auth or use their own JWT middleware
-            const isPublicEndpoint = req.path === '/health' ||
+            const isPublicEndpoint = req.path === '/health' || req.path === '/ready' ||
                 req.path.startsWith('/v1/auth/') ||
                 req.path === '/v1/webhooks/stripe' ||
                 req.path === '/v1/me' ||

package/dist/server/middleware/rate-limit.js

@@ -75,6 +75,7 @@ const TIER_BURST_LIMITS = {
  */
 const EXEMPT_PATHS = [
     '/health',
+    '/ready',
     '/openapi.json',
     '/openapi.yaml',
     '/docs',

package/dist/server/routes/fetch-queue.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Queue-backed /v1/fetch and /v1/render endpoints.
+ *
+ * Used when API_MODE=queue (microservices mode).
+ * Instead of calling peel() directly, jobs are enqueued in Bull
+ * and results are polled from Redis via GET /v1/jobs/:id.
+ *
+ * POST /v1/fetch  → enqueue in webpeel:fetch queue  → return { jobId, status }
+ * POST /v1/render → enqueue in webpeel:render queue → return { jobId, status }
+ * GET  /v1/jobs/:id → return job status + result from Redis
+ */
+import { Router } from 'express';
+export declare function createQueueFetchRouter(): Router;

package/dist/server/routes/fetch-queue.js

@@ -0,0 +1,231 @@
+/**
+ * Queue-backed /v1/fetch and /v1/render endpoints.
+ *
+ * Used when API_MODE=queue (microservices mode).
+ * Instead of calling peel() directly, jobs are enqueued in Bull
+ * and results are polled from Redis via GET /v1/jobs/:id.
+ *
+ * POST /v1/fetch  → enqueue in webpeel:fetch queue  → return { jobId, status }
+ * POST /v1/render → enqueue in webpeel:render queue → return { jobId, status }
+ * GET  /v1/jobs/:id → return job status + result from Redis
+ */
+import { Router } from 'express';
+import { randomUUID } from 'crypto';
+import { validateUrlForSSRF, SSRFError } from '../middleware/url-validator.js';
+import { getFetchQueue, getRenderQueue, RESULT_KEY_PREFIX, } from '../bull-queues.js';
+// @ts-ignore — ioredis CJS/ESM interop
+import IoRedisModule from "ioredis";
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const IoRedis = IoRedisModule.default ?? IoRedisModule;
+// ─── Redis client for result reads ──────────────────────────────────────────
+function buildRedisClient() {
+    const url = process.env.REDIS_URL || 'redis://redis:6379';
+    const password = process.env.REDIS_PASSWORD || undefined;
+    try {
+        const parsed = new URL(url);
+        return new IoRedis({
+            host: parsed.hostname,
+            port: parseInt(parsed.port || '6379', 10),
+            password,
+            db: parseInt(parsed.pathname?.slice(1) || '0', 10) || 0,
+            lazyConnect: true,
+            maxRetriesPerRequest: 3,
+        });
+    }
+    catch {
+        return new IoRedis({ host: 'redis', port: 6379, password, lazyConnect: true, maxRetriesPerRequest: 3 });
+    }
+}
+let _redis = null;
+function getRedis() {
+    if (!_redis)
+        _redis = buildRedisClient();
+    return _redis;
+}
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+async function getJobResult(jobId) {
+    const raw = await getRedis().get(`${RESULT_KEY_PREFIX}${jobId}`);
+    if (!raw)
+        return null;
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+}
+function validateUrl(url, res, requestId) {
+    if (!url || typeof url !== 'string') {
+        res.status(400).json({
+            success: false,
+            error: {
+                type: 'invalid_request',
+                message: 'Missing or invalid "url" parameter.',
+                hint: 'Send JSON: { "url": "https://example.com" }',
+                docs: 'https://webpeel.dev/docs/api-reference#fetch',
+            },
+            requestId,
+        });
+        return null;
+    }
+    if (url.length > 2048) {
+        res.status(400).json({
+            success: false,
+            error: { type: 'invalid_url', message: 'URL too long (max 2048 characters)' },
+            requestId,
+        });
+        return null;
+    }
+    try {
+        new URL(url);
+    }
+    catch {
+        res.status(400).json({
+            success: false,
+            error: {
+                type: 'invalid_url',
+                message: 'Invalid URL format',
+                hint: 'Ensure the URL includes a scheme (https://) and a valid hostname',
+                docs: 'https://webpeel.dev/docs/api-reference#fetch',
+            },
+            requestId,
+        });
+        return null;
+    }
+    try {
+        validateUrlForSSRF(url);
+    }
+    catch (e) {
+        if (e instanceof SSRFError) {
+            res.status(400).json({
+                success: false,
+                error: {
+                    type: 'forbidden_url',
+                    message: 'Cannot fetch localhost, private networks, or non-HTTP URLs',
+                },
+                requestId,
+            });
+            return null;
+        }
+        throw e;
+    }
+    return url;
+}
+// ─── Router factory ──────────────────────────────────────────────────────────
+export function createQueueFetchRouter() {
+    const router = Router();
+    /**
+     * POST /v1/fetch  — enqueue HTTP fetch job
+     * POST /v1/render — enqueue browser render job
+     * These are the queue-mode replacements for the direct peel() calls.
+     */
+    async function handleEnqueue(req, res, renderMode) {
+        const requestId = req.requestId || randomUUID();
+        const url = validateUrl(req.body?.url, res, requestId);
+        if (!url)
+            return;
+        const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!userId) {
+            res.status(401).json({
+                success: false,
+                error: {
+                    type: 'unauthorized',
+                    message: 'API key required. Get one free at https://app.webpeel.dev/keys',
+                    docs: 'https://webpeel.dev/docs/errors#unauthorized',
+                },
+                requestId,
+            });
+            return;
+        }
+        const jobId = randomUUID();
+        const payload = {
+            jobId,
+            url,
+            render: renderMode,
+            format: req.body.format || 'markdown',
+            wait: req.body.wait,
+            maxTokens: req.body.maxTokens,
+            budget: req.body.budget,
+            stealth: req.body.stealth,
+            screenshot: req.body.screenshot,
+            fullPage: req.body.fullPage,
+            selector: req.body.selector,
+            exclude: req.body.exclude,
+            includeTags: req.body.includeTags,
+            excludeTags: req.body.excludeTags,
+            images: req.body.images,
+            actions: req.body.actions,
+            timeout: req.body.timeout,
+            lite: req.body.lite,
+            raw: req.body.raw,
+            readable: req.body.readable,
+            question: req.body.question,
+            userId,
+        };
+        // Write initial queued status to Redis immediately so polling works right away
+        await getRedis().set(`${RESULT_KEY_PREFIX}${jobId}`, JSON.stringify({ status: 'queued' }), 'EX', 86_400);
+        // Enqueue in the appropriate Bull queue
+        const queue = renderMode ? getRenderQueue() : getFetchQueue();
+        await queue.add(payload, {
+            jobId, // use our own UUID as Bull job id for easy lookup
+        });
+        res.status(202).json({
+            success: true,
+            jobId,
+            status: 'queued',
+            pollUrl: `/v1/jobs/${jobId}`,
+        });
+    }
+    router.post('/v1/fetch', (req, res) => void handleEnqueue(req, res, false));
+    router.post('/v1/render', (req, res) => void handleEnqueue(req, res, true));
+    /**
+     * GET /v1/jobs/:id — return job status + result (or error)
+     * This endpoint is used regardless of whether queue mode is enabled.
+     * When a job is complete, `result` contains the full peel() output.
+     */
+    router.get('/v1/jobs/:id', async (req, res) => {
+        const { id } = req.params;
+        const requestId = req.requestId || randomUUID();
+        if (!id || typeof id !== 'string') {
+            res.status(400).json({
+                success: false,
+                error: { type: 'invalid_request', message: 'Missing job id' },
+                requestId,
+            });
+            return;
+        }
+        try {
+            const job = await getJobResult(id);
+            if (!job) {
+                res.status(404).json({
+                    success: false,
+                    error: {
+                        type: 'not_found',
+                        message: 'Job not found or expired',
+                        hint: 'Jobs expire after 24h. Check the jobId.',
+                    },
+                    requestId,
+                });
+                return;
+            }
+            const statusCode = job.status === 'failed' ? 200 : 200; // always 200 for polling
+            res.status(statusCode).json({
+                success: true,
+                jobId: id,
+                status: job.status,
+                result: job.result,
+                error: job.error,
+                startedAt: job.startedAt,
+                completedAt: job.completedAt,
+            });
+        }
+        catch (err) {
+            res.status(500).json({
+                success: false,
+                error: { type: 'internal_error', message: 'Failed to retrieve job result' },
+                requestId,
+            });
+        }
+    });
+    return router;
+}

package/dist/server/routes/health.d.ts

@@ -1,7 +1,11 @@
 /**
- * Health check endpoint
+ * Health check endpoints
  * NOTE: This route is mounted BEFORE auth/rate-limit middleware in app.ts
  * so it's never blocked by rate limiting (Render hits it every ~30s).
+ *
+ * GET /health  — liveness probe: always returns 200 if process is alive
+ * GET /ready   — readiness probe: checks DB + job queue; returns 503 if not ready
  */
 import { Router } from 'express';
-export declare function createHealthRouter(): Router;
+import type pg from 'pg';
+export declare function createHealthRouter(pool?: pg.Pool | null): Router;

package/dist/server/routes/health.js

@@ -1,7 +1,10 @@
 /**
- * Health check endpoint
+ * Health check endpoints
  * NOTE: This route is mounted BEFORE auth/rate-limit middleware in app.ts
  * so it's never blocked by rate limiting (Render hits it every ~30s).
+ *
+ * GET /health  — liveness probe: always returns 200 if process is alive
+ * GET /ready   — readiness probe: checks DB + job queue; returns 503 if not ready
  */
 import { Router } from 'express';
 import { readFileSync } from 'fs';
@@ -23,8 +26,12 @@ catch {
     }
     catch { /* keep 'unknown' */ }
 }
-export function createHealthRouter() {
+export function createHealthRouter(pool) {
     const router = Router();
+    // ------------------------------------------------------------------
+    // GET /health — liveness probe
+    // K8s: if this fails, pod is restarted
+    // ------------------------------------------------------------------
     router.get('/health', (_req, res) => {
         const uptime = Math.floor((Date.now() - startTime) / 1000);
         const fetchStats = fetchCache.stats();
@@ -46,5 +53,66 @@ export function createHealthRouter() {
             },
         });
     });
+    // ------------------------------------------------------------------
+    // GET /ready — readiness probe
+    // K8s: if this fails, pod is removed from service endpoints (no traffic)
+    // Checks: database connectivity + queue (job table) reachability
+    // ------------------------------------------------------------------
+    router.get('/ready', async (_req, res) => {
+        const checks = {};
+        let allOk = true;
+        // --- Database check ---
+        if (pool) {
+            const t0 = Date.now();
+            try {
+                await pool.query('SELECT 1');
+                checks.database = { ok: true, latencyMs: Date.now() - t0 };
+            }
+            catch (err) {
+                checks.database = { ok: false, latencyMs: Date.now() - t0, error: err?.message ?? 'unknown' };
+                allOk = false;
+            }
+        }
+        else {
+            // No pool configured (in-memory mode / local dev without DATABASE_URL)
+            checks.database = { ok: true, latencyMs: 0 };
+        }
+        // --- Job queue check (probe the jobs table via DATABASE_URL directly) ---
+        if (process.env.DATABASE_URL) {
+            const t0 = Date.now();
+            try {
+                // Reuse the same pool if available, or do a lightweight table existence check
+                if (pool) {
+                    await pool.query('SELECT COUNT(*) FROM jobs WHERE status = $1 LIMIT 1', ['queued']);
+                    checks.queue = { ok: true, latencyMs: Date.now() - t0 };
+                }
+                else {
+                    checks.queue = { ok: true, latencyMs: 0 };
+                }
+            }
+            catch (err) {
+                // Table may not exist in early boot; treat as non-fatal
+                const msg = err?.message ?? '';
+                if (msg.includes('relation "jobs" does not exist')) {
+                    checks.queue = { ok: true, latencyMs: Date.now() - t0 };
+                }
+                else {
+                    checks.queue = { ok: false, latencyMs: Date.now() - t0, error: msg };
+                    allOk = false;
+                }
+            }
+        }
+        else {
+            checks.queue = { ok: true, latencyMs: 0 };
+        }
+        const status = allOk ? 200 : 503;
+        res.status(status).json({
+            status: allOk ? 'ready' : 'not_ready',
+            version,
+            uptime: Math.floor((Date.now() - startTime) / 1000),
+            timestamp: new Date().toISOString(),
+            checks,
+        });
+    });
     return router;
 }

package/dist/server/routes/search.js

@@ -9,6 +9,7 @@ import { peel } from '../../index.js';
 import { simpleFetch } from '../../core/fetcher.js';
 import { searchCache } from '../../core/fetch-cache.js';
 import { getSearchProvider, getBestSearchProvider, } from '../../core/search-provider.js';
+import { getSourceCredibility } from '../../core/source-credibility.js';
 export function createSearchRouter(authStore) {
     const router = Router();
     // LRU cache: 15 minute TTL, max 500 entries, 50MB total size
@@ -210,6 +211,19 @@ export function createSearchRouter(authStore) {
                         }
                     }
                 }
+                // Add credibility scores and sort by trustworthiness
+                const tierOrder = { official: 0, verified: 1, general: 2 };
+                results = results
+                    .map(r => {
+                    const cred = getSourceCredibility(r.url);
+                    return { ...r, credibility: cred };
+                })
+                    .sort((a, b) => {
+                    const aTier = tierOrder[a.credibility?.tier || 'general'] ?? 2;
+                    const bTier = tierOrder[b.credibility?.tier || 'general'] ?? 2;
+                    return aTier - bTier; // Official first, then verified, then general
+                })
+                    .map((r, i) => ({ ...r, rank: i + 1 }));
                 data.web = results;
             }
             // Fetch news results (DDG only — Brave news is not supported via HTML scraping)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.53",
+  "version": "0.21.57",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",
@@ -96,10 +96,12 @@
   ],
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.0.4",
+    "bull": "^4.16.5",
     "cheerio": "^1.0.0",
     "cloakbrowser": "^0.1.8",
     "commander": "^12.0.0",
     "cycletls": "^2.0.5",
+    "ioredis": "^5.10.0",
     "lru-cache": "^11.0.2",
     "mammoth": "^1.11.0",
     "nodemailer": "^8.0.1",