scraply 1.0.18 → 1.0.19

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.18",
+  "version": "1.0.19",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/src/utils/crawl/url/fetch.js

@@ -19,12 +19,9 @@ export async function fetchURL(url, retries = 2) {
       };
     };
 
-    return {
-      data: response.data,
-      status: response.status
-    };
+    return { data: response.data, status: response.status };
   } catch (error) {
-    if (retries > 0 && shouldRetry(error)) {
+    if (retries > 0 && (await shouldRetry(error))) {
       const retryCount = CONFIG.CRAWLER.MAX_RETRIES - retries + 1;
       console.log(`Retrying (${retryCount}/${CONFIG.CRAWLER.MAX_RETRIES}) -> ${url}`);
       
@@ -32,11 +29,14 @@ export async function fetchURL(url, retries = 2) {
       
       return fetchURL(url, retries - 1);
     }
+
+    // If still 429 after retries, exit with configured code
+    if (error.response?.status === 429) {
+      console.log(`Force exiting with code ${CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT} after retries...`);
+      process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT);
+    }
     
     console.error(`Failed to fetch ${url} -> ${error.message}`);
-    return {
-      error: error.message,
-      status: error.response?.status
-    };
+    return { error: error.message, status: error.response?.status };
   };
 };

package/src/utils/crawl/url/handlers.js

@@ -11,25 +11,27 @@ export const shouldRetry = async (error) => {
   const rateLimitReset = headers?.['x-ratelimit-reset'];
 
   if (status === 429) {
-    let waitTime = null;
-
-    if (retryAfter) {
-      waitTime = isNaN(retryAfter)
-        ? Math.ceil((new Date(retryAfter).getTime() - Date.now()) / 1000) // HTTP date
-        : parseInt(retryAfter, 10); // Seconds
-      console.log(`Rate limited. Retrying after ${waitTime} seconds...`);
-    } else if (rateLimitReset) {
-      waitTime = Math.max(parseInt(rateLimitReset, 10) - Math.floor(Date.now() / 1000), 0);
-      console.log(`Rate limited. Retrying after ${waitTime} seconds...`);
-    } else {
-      waitTime = CONFIG.CRAWLER.CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS / 1000;
-      console.log(`Rate limited. No 'retry-after' or 'x-ratelimit-reset' headers found. Falling back to ${waitTime} seconds...`);
-    }
+    console.log(`RATE LIMIT Detected.`);
 
     if (CONFIG.CRAWLER.EXIT_ON_RATE_LIMIT) {
-      console.log(`Exiting due to rate limit.`);
-      process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT); // GitHub Actions Docker uses values ranged from 0 to 255, so any bigger value will be % 256!
+      console.log(`Force exiting with code ${CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT}...`);
+      process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT); // GitHub Actions Docker uses values ranged from 0 to 255. Any bigger value will be % 256
     } else {
+      let waitTime = null;
+
+      if (retryAfter) {
+        waitTime = isNaN(retryAfter)
+          ? Math.ceil((new Date(retryAfter).getTime() - Date.now()) / 1000) // HTTP date
+          : parseInt(retryAfter, 10); // Seconds
+        console.log(`'retry-after' headers found.`);
+      } else if (rateLimitReset) {
+        waitTime = Math.max(parseInt(rateLimitReset, 10) - Math.floor(Date.now() / 1000), 0);
+        console.log(`'x-ratelimit-reset' headers found.`);
+      } else {
+        waitTime = CONFIG.CRAWLER.CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS / 1000;
+        console.log(`No 'retry-after' or 'x-ratelimit-reset' headers found. Using fallback delay.`);
+      }
+      console.log(`Retrying after ${waitTime} seconds...`);
       await delay(waitTime * 1000);
     }
   }