scraply 1.0.19 → 1.0.20

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.19",
+  "version": "1.0.20",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/readme.md

@@ -70,15 +70,16 @@ CRAWLER: {
     'aside',
     'button'
   ],
-  RETRY_STATUS_CODES: [408, 429, 500, 502, 503, 504],
-  REQUEST_TIMEOUT: 4000,
-  MAX_REDIRECTS: 3,
-  MAX_RETRIES: 2,
+  RETRY_STATUS_CODES: [408, 500, 502, 503, 504],
+  REQUEST_TIMEOUT: 3000,
+  MAX_REDIRECTS: 2,
+  MAX_CONTENT_LENGTH: 20 * 1024 * 1024, // 20MB
+  MAX_RETRIES: 1,
   CRAWL_DELAY_MS: 200,
-  CRAWL_ERROR_RETRY_DELAY_MS: 800,
+  CRAWL_ERROR_RETRY_DELAY_MS: 1000,
   CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS: 60000,
-  EXIT_CODE_RATE_LIMIT: 10,
-  EXIT_ON_RATE_LIMIT: true
+  EXIT_ON_RATE_LIMIT: true, // If true, forces exit instantly. If false, only exits after retries (if still 429)
+  EXIT_CODE_RATE_LIMIT: 10
 },
 
 DATA_FORMATTER: {

package/src/defaultConfig.js

@@ -35,14 +35,15 @@ export const DEFAULT_CONFIG = {
       'aside',
       'button'
     ],
-    RETRY_STATUS_CODES: [408, 429, 500, 502, 503, 504],
-    REQUEST_TIMEOUT: 4000,
-    MAX_REDIRECTS: 3,
-    MAX_RETRIES: 2,
+    RETRY_STATUS_CODES: [408, 500, 502, 503, 504],
+    REQUEST_TIMEOUT: 3000,
+    MAX_REDIRECTS: 2,
+    MAX_CONTENT_LENGTH: 20 * 1024 * 1024, // 20MB
+    MAX_RETRIES: 1,
     CRAWL_DELAY_MS: 200,
-    CRAWL_ERROR_RETRY_DELAY_MS: 800,
+    CRAWL_ERROR_RETRY_DELAY_MS: 1000,
     CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS: 60000,
-    EXIT_ON_RATE_LIMIT: true,
+    EXIT_ON_RATE_LIMIT: true, // If true, forces exit instantly. If false, only exits after retries (if still 429)
     EXIT_CODE_RATE_LIMIT: 10
   },
 

package/src/utils/crawl/url/fetch.js

@@ -6,17 +6,20 @@ export async function fetchURL(url, retries = 2) {
   try {
     const response = await axios.get(url, {
       timeout: CONFIG.CRAWLER.REQUEST_TIMEOUT,
-      maxRedirects: CONFIG.CRAWLER.MAX_REDIRECTS
+      maxRedirects: CONFIG.CRAWLER.MAX_REDIRECTS,
+      maxContentLength: CONFIG.CRAWLER.MAX_CONTENT_LENGTH
     });
 
     const { 'content-type': contentType } = response.headers;
 
+    if (!contentType) {
+      console.log(`Missing Content-Type header for ${url}`);
+      return { error: `Missing Content-Type header`, status: response.status };
+    };
+
     // Validate content type
     if (!CONFIG.CRAWLER.ALLOWED_CONTENT_TYPES.some(type => contentType.includes(type))) {
-      return {
-        error: `Content-Type ${contentType} is not allowed.`,
-        status: response.status
-      };
+      return { error: `Content-Type ${contentType} is not allowed.`, status: response.status };
     };
 
     return { data: response.data, status: response.status };
@@ -32,7 +35,7 @@ export async function fetchURL(url, retries = 2) {
 
     // If still 429 after retries, exit with configured code
     if (error.response?.status === 429) {
-      console.log(`Force exiting with code ${CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT} after retries...`);
+      console.log(`Force exiting with code ${CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT} (after retries)...`);
       process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT);
     }