webpeel 0.21.63 → 0.21.64

package/dist/core/http-fetch.js

@@ -559,10 +559,14 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
         try {
             const requestHeaders = { ...mergedHeaders };
             const validators = getConditionalValidators(currentUrl);
-            if (validators?.etag && !hasHeader(requestHeaders, 'if-none-match')) {
+            // Only send conditional headers if we actually have the cached body
+            // In server/worker mode, the in-memory cache may have been cleared (pod restart)
+            // and sending If-None-Match without a cached body would cause a 304 crash
+            const cachedBody = getCachedResultFor304(currentUrl, url);
+            if (validators?.etag && cachedBody && !hasHeader(requestHeaders, 'if-none-match')) {
                 requestHeaders['If-None-Match'] = validators.etag;
             }
-            if (validators?.lastModified && !hasHeader(requestHeaders, 'if-modified-since')) {
+            if (validators?.lastModified && cachedBody && !hasHeader(requestHeaders, 'if-modified-since')) {
                 requestHeaders['If-Modified-Since'] = validators.lastModified;
             }
             // Use proxy if provided or auto-selected, otherwise use shared connection pool

package/dist/server/routes/fetch-queue.js

@@ -176,6 +176,81 @@ export function createQueueFetchRouter() {
             pollUrl: `/v1/jobs/${jobId}`,
         });
     }
+    /**
+     * GET/POST /v1/fetch/sync — Synchronous fetch, no queue
+     * Returns content inline (no jobId/polling). Much faster for simple pages.
+     * Timeout: 25s max. No fallback to queue — fails fast if timeout exceeded.
+     */
+    async function handleSyncFetch(req, res) {
+        const requestId = req.requestId || randomUUID();
+        const url = validateUrl(req.body?.url || req.query?.url, res, requestId);
+        if (!url)
+            return;
+        const userId = req.auth?.keyInfo?.accountId || req.user?.userId;
+        if (!userId) {
+            res.status(401).json({
+                success: false,
+                error: { type: 'unauthorized', message: 'API key required.' },
+                requestId,
+            });
+            return;
+        }
+        try {
+            // Import peel dynamically to avoid circular deps
+            const { peel } = await import('../../index.js');
+            const options = {
+                format: req.body?.format || req.query?.format || 'markdown',
+                render: req.body?.render === true || req.query?.render === 'true',
+                stealth: req.body?.stealth === true || req.query?.stealth === 'true',
+                budget: req.body?.budget ? Number(req.body.budget) : (req.query?.budget ? Number(req.query.budget) : undefined),
+                selector: req.body?.selector || req.query?.selector,
+                readable: req.body?.readable === true || req.query?.readable === 'true',
+                wait: req.body?.wait ? Number(req.body.wait) : (req.query?.wait ? Number(req.query.wait) : undefined),
+                question: req.body?.question || req.query?.question,
+                timeout: 25000, // 25s max (leave 5s buffer for response)
+            };
+            const result = await peel(url, options);
+            res.json({
+                success: true,
+                ...result,
+                requestId,
+                mode: 'sync',
+            });
+        }
+        catch (err) {
+            const statusCode = err.statusCode || 500;
+            res.status(statusCode >= 400 && statusCode < 600 ? statusCode : 500).json({
+                success: false,
+                error: {
+                    type: err.errorType || 'fetch_error',
+                    message: err.message || 'Fetch failed',
+                },
+                requestId,
+            });
+        }
+    }
+    router.get('/v1/fetch/sync', (req, res) => {
+        // Map query params to body
+        req.body = req.body || {};
+        if (req.query.url)
+            req.body.url = req.query.url;
+        if (req.query.format)
+            req.body.format = req.query.format;
+        if (req.query.render)
+            req.body.render = req.query.render === 'true';
+        if (req.query.stealth)
+            req.body.stealth = req.query.stealth === 'true';
+        if (req.query.budget)
+            req.body.budget = Number(req.query.budget);
+        if (req.query.selector)
+            req.body.selector = req.query.selector;
+        if (req.query.readable)
+            req.body.readable = req.query.readable === 'true';
+        if (req.query.question)
+            req.body.question = req.query.question;
+        void handleSyncFetch(req, res);
+    });
+    router.post('/v1/fetch/sync', (req, res) => void handleSyncFetch(req, res));
     // GET /v1/fetch?url=...  — CLI and backward-compatible GET requests
     // Maps query params into req.body so handleEnqueue works uniformly
     router.get('/v1/fetch', (req, res) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpeel",
-  "version": "0.21.63",
+  "version": "0.21.64",
   "description": "Fast web fetcher for AI agents - stealth mode, crawl mode, page actions, structured extraction, PDF parsing, smart escalation from simple HTTP to headless browser",
   "author": "Jake Liu",
   "license": "AGPL-3.0-only",
@@ -59,6 +59,7 @@
     "prepublishOnly": "bash scripts/pre-publish.sh",
     "serve": "node dist/server/app.js",
     "mcp": "node dist/mcp/server.js",
+    "preversion": "npm run build && npm test && bash scripts/pre-publish-gate.sh",
     "version": "bash scripts/postversion.sh"
   },
   "repository": {