webpeel 0.21.0 → 0.21.1

package/dist/core/dns-cache.js

@@ -40,6 +40,21 @@ const DNS_WARMUP_DOMAINS = [
     'tools.ietf.org',
     'unicode.org',
     'www.bbc.com',
+    'bbc.co.uk',
+    'stripe.com',
+    'docs.stripe.com',
+    'vuejs.org',
+    'angular.io',
+    'www.washingtonpost.com',
+    'www.theguardian.com',
+    'techcrunch.com',
+    'www.wired.com',
+    'arstechnica.com',
+    'docs.google.com',
+    'drive.google.com',
+    'www.notion.so',
+    'www.producthunt.com',
+    'www.crunchbase.com',
     'news.google.com',
     'www.youtube.com',
     'example.com',

package/dist/core/http-fetch.js

@@ -42,11 +42,13 @@ const HTTP_STATUS_TEXT = {
 // ── HTTP connection pool ──────────────────────────────────────────────────────
 function createHttpPool() {
     return new Agent({
-        connections: 20,
-        pipelining: 6,
+        connections: 50,
+        pipelining: 10,
         keepAliveTimeout: 60000,
         keepAliveMaxTimeout: 60000,
         allowH2: true,
+        headersTimeout: 10000,
+        bodyTimeout: 30000,
         connect: {
             lookup: cachedLookup,
         },

package/dist/core/pipeline.d.ts

@@ -75,6 +75,8 @@ export interface PipelineContext {
     domainApiHandled?: boolean;
     /** True when server returned pre-rendered markdown (Content-Type: text/markdown) */
     serverMarkdown?: boolean;
+    /** True when HTTP fetch completed in < 500ms — enables fast path (skip challenge detection) */
+    fastPath?: boolean;
     /** Non-fatal warnings accumulated during the pipeline run */
     warnings: string[];
     /** Raw HTML size in characters (measured from fetched content before any conversion) */

package/dist/core/pipeline.js

@@ -399,7 +399,17 @@ export async function fetchContent(ctx) {
         }
         throw fetchError;
     }
-    ctx.timer.end('fetch');
+    const fetchDuration = ctx.timer.end('fetch');
+    // Fast path: if a plain HTTP fetch completed quickly with real HTML content,
+    // mark it so post-processing can skip expensive heuristics (challenge detection).
+    // Only applies to non-browser fetches that succeeded with HTML content.
+    if (fetchDuration < 500 &&
+        !ctx.render &&
+        fetchResult.statusCode === 200 &&
+        (fetchResult.contentType || '').includes('html') &&
+        (fetchResult.html?.length || 0) > 200) {
+        ctx.fastPath = true;
+    }
     // Auto-scroll to load lazy content, then grab fresh HTML
     if (needsAutoScroll && fetchResult.page) {
         try {
@@ -927,7 +937,9 @@ export async function postProcess(ctx) {
     // === Challenge / bot-protection page detection ===
     // If the extracted content looks like a challenge page (not real content),
     // mark it and try the search-as-proxy fallback to get the real info.
-    if (ctx.content && ctx.content.length < 2000) {
+    // Fast path: skip this check for HTTP fetches that completed in < 500ms —
+    // a fast successful response is virtually never a challenge page.
+    if (!ctx.fastPath && ctx.content && ctx.content.length < 2000) {
         const lowerContent = ctx.content.toLowerCase();
         const challengeSignals = [
             'please verify you are a human',

package/dist/server/routes/fetch.js

@@ -229,7 +229,10 @@ export function createFetchRouter(authStore) {
                     const cacheAge = Date.now() - cached.timestamp;
                     if (cacheAge < maxAgeMs && cacheAge < cacheTtlMs) {
                         res.setHeader('X-Cache', 'HIT');
+                        res.setHeader('X-Cache-Status', 'HIT');
                         res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
+                        // Cache-Control: allow Cloudflare edge to cache successful GET responses
+                        res.setHeader('Cache-Control', 'public, s-maxage=60, stale-while-revalidate=300');
                         if (wantsEnvelope(req)) {
                             successResponse(res, cached.result, {
                                 requestId: req.requestId,
@@ -467,9 +470,19 @@ export function createFetchRouter(authStore) {
                 : undefined;
             // Add usage headers (kept for backward compat; also surfaced in envelope metadata)
             res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Cache-Status', 'MISS');
             res.setHeader('X-Credits-Used', '1');
             res.setHeader('X-Processing-Time', elapsed.toString());
             res.setHeader('X-Fetch-Type', fetchType);
+            // Cache-Control: allow Cloudflare edge to cache successful GET responses for 60s
+            res.setHeader('Cache-Control', 'public, s-maxage=60, stale-while-revalidate=300');
+            // Response timing headers — let customers see exactly where time is spent
+            const timingFetch = result.timing?.fetch ?? 0;
+            const timingParse = (result.timing?.convert ?? 0) + (result.timing?.metadata ?? 0) + (result.timing?.prune ?? 0);
+            res.setHeader('X-Response-Time', `${elapsed}ms`);
+            res.setHeader('X-Fetch-Time', `${timingFetch}ms`);
+            res.setHeader('X-Parse-Time', `${timingParse}ms`);
+            res.setHeader('Server-Timing', `fetch;dur=${timingFetch}, parse;dur=${timingParse}, total;dur=${elapsed}`);
             // Build response — extend result with optional answer/summary fields
             const getResponseBody = { ...result };
             if (getAnswerResult !== undefined)
@@ -676,6 +689,7 @@ export function createFetchRouter(authStore) {
                     const cacheAge = Date.now() - cached.timestamp;
                     if (cacheAge < postCacheTtlMs) {
                         res.setHeader('X-Cache', 'HIT');
+                        res.setHeader('X-Cache-Status', 'HIT');
                         res.setHeader('X-Cache-Age', Math.floor(cacheAge / 1000).toString());
                         if (wantsEnvelope(req)) {
                             successResponse(res, cached.result, {
@@ -932,9 +946,17 @@ export function createFetchRouter(authStore) {
             // --- Build response ------------------------------------------------------
             // Headers kept for backward compat; also surfaced in envelope metadata.
             res.setHeader('X-Cache', 'MISS');
+            res.setHeader('X-Cache-Status', 'MISS');
             res.setHeader('X-Credits-Used', '1');
             res.setHeader('X-Processing-Time', elapsed.toString());
             res.setHeader('X-Fetch-Type', fetchType);
+            // Response timing headers — let customers see exactly where time is spent
+            const postTimingFetch = result.timing?.fetch ?? 0;
+            const postTimingParse = (result.timing?.convert ?? 0) + (result.timing?.metadata ?? 0) + (result.timing?.prune ?? 0);
+            res.setHeader('X-Response-Time', `${elapsed}ms`);
+            res.setHeader('X-Fetch-Time', `${postTimingFetch}ms`);
+            res.setHeader('X-Parse-Time', `${postTimingParse}ms`);
+            res.setHeader('Server-Timing', `fetch;dur=${postTimingFetch}, parse;dur=${postTimingParse}, total;dur=${elapsed}`);
             const responseBody = { ...result };
             if (jsonData !== undefined) {
                 responseBody.json = jsonData;

package/dist/server/routes/reader.js

@@ -96,6 +96,11 @@ export function createReaderRouter() {
                 selector: targetSelector,
                 waitSelector: waitForSelector,
             });
+            // Cache-Control: this endpoint is public and heavily cacheable.
+            // Cloudflare edge caches for 2 min; serves stale for up to 10 min while revalidating.
+            res.setHeader('Cache-Control', 'public, s-maxage=120, stale-while-revalidate=600');
+            // Vary on Accept so different content-type representations are cached separately.
+            res.setHeader('Vary', 'Accept');
             // Return based on format
             const responseFormat = format.toLowerCase();
             if (responseFormat === 'text') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.0",
+  "version": "0.21.1",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",