scraply 1.0.18 → 1.0.20

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.18",
+  "version": "1.0.20",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/readme.md

@@ -70,15 +70,16 @@ CRAWLER: {
     'aside',
     'button'
   ],
-  RETRY_STATUS_CODES: [408, 429, 500, 502, 503, 504],
-  REQUEST_TIMEOUT: 4000,
-  MAX_REDIRECTS: 3,
-  MAX_RETRIES: 2,
+  RETRY_STATUS_CODES: [408, 500, 502, 503, 504],
+  REQUEST_TIMEOUT: 3000,
+  MAX_REDIRECTS: 2,
+  MAX_CONTENT_LENGTH: 20 * 1024 * 1024, // 20MB
+  MAX_RETRIES: 1,
   CRAWL_DELAY_MS: 200,
-  CRAWL_ERROR_RETRY_DELAY_MS: 800,
+  CRAWL_ERROR_RETRY_DELAY_MS: 1000,
   CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS: 60000,
-  EXIT_CODE_RATE_LIMIT: 10,
-  EXIT_ON_RATE_LIMIT: true
+  EXIT_ON_RATE_LIMIT: true, // If true, forces exit instantly. If false, only exits after retries (if still 429)
+  EXIT_CODE_RATE_LIMIT: 10
 },
 
 DATA_FORMATTER: {

package/src/defaultConfig.js

@@ -35,14 +35,15 @@ export const DEFAULT_CONFIG = {
       'aside',
       'button'
     ],
-    RETRY_STATUS_CODES: [408, 429, 500, 502, 503, 504],
-    REQUEST_TIMEOUT: 4000,
-    MAX_REDIRECTS: 3,
-    MAX_RETRIES: 2,
+    RETRY_STATUS_CODES: [408, 500, 502, 503, 504],
+    REQUEST_TIMEOUT: 3000,
+    MAX_REDIRECTS: 2,
+    MAX_CONTENT_LENGTH: 20 * 1024 * 1024, // 20MB
+    MAX_RETRIES: 1,
     CRAWL_DELAY_MS: 200,
-    CRAWL_ERROR_RETRY_DELAY_MS: 800,
+    CRAWL_ERROR_RETRY_DELAY_MS: 1000,
     CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS: 60000,
-    EXIT_ON_RATE_LIMIT: true,
+    EXIT_ON_RATE_LIMIT: true, // If true, forces exit instantly. If false, only exits after retries (if still 429)
     EXIT_CODE_RATE_LIMIT: 10
   },
 

package/src/utils/crawl/url/fetch.js

@@ -6,25 +6,25 @@ export async function fetchURL(url, retries = 2) {
   try {
     const response = await axios.get(url, {
       timeout: CONFIG.CRAWLER.REQUEST_TIMEOUT,
-      maxRedirects: CONFIG.CRAWLER.MAX_REDIRECTS
+      maxRedirects: CONFIG.CRAWLER.MAX_REDIRECTS,
+      maxContentLength: CONFIG.CRAWLER.MAX_CONTENT_LENGTH
     });
 
     const { 'content-type': contentType } = response.headers;
 
+    if (!contentType) {
+      console.log(`Missing Content-Type header for ${url}`);
+      return { error: `Missing Content-Type header`, status: response.status };
+    };
+
     // Validate content type
     if (!CONFIG.CRAWLER.ALLOWED_CONTENT_TYPES.some(type => contentType.includes(type))) {
-      return {
-        error: `Content-Type ${contentType} is not allowed.`,
-        status: response.status
-      };
+      return { error: `Content-Type ${contentType} is not allowed.`, status: response.status };
     };
 
-    return {
-      data: response.data,
-      status: response.status
-    };
+    return { data: response.data, status: response.status };
   } catch (error) {
-    if (retries > 0 && shouldRetry(error)) {
+    if (retries > 0 && (await shouldRetry(error))) {
       const retryCount = CONFIG.CRAWLER.MAX_RETRIES - retries + 1;
       console.log(`Retrying (${retryCount}/${CONFIG.CRAWLER.MAX_RETRIES}) -> ${url}`);
       
@@ -32,11 +32,14 @@ export async function fetchURL(url, retries = 2) {
       
       return fetchURL(url, retries - 1);
     }
+
+    // If still 429 after retries, exit with configured code
+    if (error.response?.status === 429) {
+      console.log(`Force exiting with code ${CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT} (after retries)...`);
+      process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT);
+    }
     
     console.error(`Failed to fetch ${url} -> ${error.message}`);
-    return {
-      error: error.message,
-      status: error.response?.status
-    };
+    return { error: error.message, status: error.response?.status };
   };
 };

package/src/utils/crawl/url/handlers.js

@@ -11,25 +11,27 @@ export const shouldRetry = async (error) => {
   const rateLimitReset = headers?.['x-ratelimit-reset'];
 
   if (status === 429) {
-    let waitTime = null;
-
-    if (retryAfter) {
-      waitTime = isNaN(retryAfter)
-        ? Math.ceil((new Date(retryAfter).getTime() - Date.now()) / 1000) // HTTP date
-        : parseInt(retryAfter, 10); // Seconds
-      console.log(`Rate limited. Retrying after ${waitTime} seconds...`);
-    } else if (rateLimitReset) {
-      waitTime = Math.max(parseInt(rateLimitReset, 10) - Math.floor(Date.now() / 1000), 0);
-      console.log(`Rate limited. Retrying after ${waitTime} seconds...`);
-    } else {
-      waitTime = CONFIG.CRAWLER.CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS / 1000;
-      console.log(`Rate limited. No 'retry-after' or 'x-ratelimit-reset' headers found. Falling back to ${waitTime} seconds...`);
-    }
+    console.log(`RATE LIMIT Detected.`);
 
     if (CONFIG.CRAWLER.EXIT_ON_RATE_LIMIT) {
-      console.log(`Exiting due to rate limit.`);
-      process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT); // GitHub Actions Docker uses values ranged from 0 to 255, so any bigger value will be % 256!
+      console.log(`Force exiting with code ${CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT}...`);
+      process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT); // GitHub Actions Docker uses values ranged from 0 to 255. Any bigger value will be % 256
     } else {
+      let waitTime = null;
+
+      if (retryAfter) {
+        waitTime = isNaN(retryAfter)
+          ? Math.ceil((new Date(retryAfter).getTime() - Date.now()) / 1000) // HTTP date
+          : parseInt(retryAfter, 10); // Seconds
+        console.log(`'retry-after' headers found.`);
+      } else if (rateLimitReset) {
+        waitTime = Math.max(parseInt(rateLimitReset, 10) - Math.floor(Date.now() / 1000), 0);
+        console.log(`'x-ratelimit-reset' headers found.`);
+      } else {
+        waitTime = CONFIG.CRAWLER.CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS / 1000;
+        console.log(`No 'retry-after' or 'x-ratelimit-reset' headers found. Using fallback delay.`);
+      }
+      console.log(`Retrying after ${waitTime} seconds...`);
       await delay(waitTime * 1000);
     }
   }