spectrawl 0.4.2 → 0.4.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spectrawl",
-  "version": "0.4.2",
+  "version": "0.4.3",
   "description": "The unified web layer for AI agents. Search (8 engines), stealth browse, auth, act on 24 platforms. Self-hosted.",
   "main": "src/index.js",
   "types": "index.d.ts",

package/src/browse/index.js

@@ -127,12 +127,21 @@ class BrowseEngine {
 
       await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 })
 
-      // Human-like delays
-      await page.waitForTimeout(800 + Math.random() * 1500)
-      await page.evaluate(() => {
-        window.scrollBy({ top: Math.floor(Math.random() * 400) + 100, behavior: 'smooth' })
-      })
-      await page.waitForTimeout(300 + Math.random() * 700)
+      if (opts.fastMode) {
+        // Crawl mode: minimal delays, just enough for lazy-load triggers
+        await page.waitForTimeout(400)
+        await page.evaluate(() => {
+          window.scrollBy({ top: 500, behavior: 'instant' })
+        })
+        await page.waitForTimeout(200)
+      } else {
+        // Normal browse: full human-like delays
+        await page.waitForTimeout(800 + Math.random() * 1500)
+        await page.evaluate(() => {
+          window.scrollBy({ top: Math.floor(Math.random() * 400) + 100, behavior: 'smooth' })
+        })
+        await page.waitForTimeout(300 + Math.random() * 700)
+      }
 
       const result = {}
 

package/src/crawl.js

@@ -260,8 +260,10 @@ class CrawlEngine {
     const totalMB = Math.floor(os.totalmem() / 1024 / 1024)
     const freeMB = Math.floor(os.freemem() / 1024 / 1024)
     const concurrency = detectConcurrency()
-    // Estimate: each page takes ~4s with stealth delays
-    const pagesPerMinute = concurrency * 15  // ~4s per page
+    // Realistic: ~0.8s per page with fast mode, limited by shared browser pipeline
+    // Concurrency helps but not linearly — shared browser bottleneck
+    const effectiveConcurrency = Math.min(concurrency, 5) // diminishing returns past 5
+    const pagesPerMinute = Math.floor(effectiveConcurrency * 30)  // ~2s effective per page with overhead
     return {
       totalRamMB: totalMB,
       freeRamMB: freeMB,
@@ -279,7 +281,8 @@ class CrawlEngine {
         _cookies: cookies,
         timeout: config.timeout,
         html: true,
-        noCache: true
+        noCache: true,
+        fastMode: true  // crawl mode: reduced delays for speed
       })
       if (result?.content) {
         const linkSource = result.html || result.content