scraply 1.0.17 → 1.0.19

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.17",
+  "version": "1.0.19",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/src/defaultConfig.js

@@ -42,8 +42,8 @@ export const DEFAULT_CONFIG = {
     CRAWL_DELAY_MS: 200,
     CRAWL_ERROR_RETRY_DELAY_MS: 800,
     CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS: 60000,
-    EXIT_CODE_RATE_LIMIT: 10,
-    EXIT_ON_RATE_LIMIT: true
+    EXIT_ON_RATE_LIMIT: true,
+    EXIT_CODE_RATE_LIMIT: 10
   },
 
   DATA_FORMATTER: {

package/src/utils/crawl/cleanHTML.js

@@ -6,9 +6,6 @@ export const cleanHTML = ($) => {
   $aux(CONFIG.CRAWLER.DOM_ELEMENTS_REMOVE.join(',')).remove();
   $aux('*').contents().filter((_, el) => el.type === 'comment').remove();
 
-  // Get the text content of the body and decode HTML entities
-  // let bodyText = he.decode($aux('body').text(), { level: 'all' });
-
   // Get the text content of the body element, ensuring spaces between child elements
   let bodyText = getTextWithSpaces($aux, $aux('body'));
 

package/src/utils/crawl/url/fetch.js

@@ -19,12 +19,9 @@ export async function fetchURL(url, retries = 2) {
       };
     };
 
-    return {
-      data: response.data,
-      status: response.status
-    };
+    return { data: response.data, status: response.status };
   } catch (error) {
-    if (retries > 0 && shouldRetry(error)) {
+    if (retries > 0 && (await shouldRetry(error))) {
       const retryCount = CONFIG.CRAWLER.MAX_RETRIES - retries + 1;
       console.log(`Retrying (${retryCount}/${CONFIG.CRAWLER.MAX_RETRIES}) -> ${url}`);
       
@@ -32,11 +29,14 @@ export async function fetchURL(url, retries = 2) {
       
       return fetchURL(url, retries - 1);
     }
+
+    // If still 429 after retries, exit with configured code
+    if (error.response?.status === 429) {
+      console.log(`Force exiting with code ${CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT} after retries...`);
+      process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT);
+    }
     
     console.error(`Failed to fetch ${url} -> ${error.message}`);
-    return {
-      error: error.message,
-      status: error.response?.status
-    };
+    return { error: error.message, status: error.response?.status };
   };
 };

package/src/utils/crawl/url/handlers.js

@@ -11,25 +11,27 @@ export const shouldRetry = async (error) => {
   const rateLimitReset = headers?.['x-ratelimit-reset'];
 
   if (status === 429) {
-    let waitTime = null;
-
-    if (retryAfter) {
-      waitTime = isNaN(retryAfter)
-        ? Math.ceil((new Date(retryAfter).getTime() - Date.now()) / 1000) // HTTP date
-        : parseInt(retryAfter, 10); // Seconds
-      console.log(`Rate limited. Retrying after ${waitTime} seconds...`);
-    } else if (rateLimitReset) {
-      waitTime = Math.max(parseInt(rateLimitReset, 10) - Math.floow(Date.now() / 1000), 0);
-      console.log(`Rate limited. Retrying after ${waitTime} seconds...`);
-    } else {
-      waitTime = CONFIG.CRAWLER.CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS / 1000;
-      console.log(`Rate limited. No 'retry-after' or 'x-ratelimit-reset' headers found. Falling back to ${waitTime} seconds...`);
-    }
+    console.log(`RATE LIMIT Detected.`);
 
     if (CONFIG.CRAWLER.EXIT_ON_RATE_LIMIT) {
-      console.log(`Exiting due to rate limit.`);
-      process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT); // GitHub Actions Docker uses values ranged from 0 to 255, so any bigger value will be % 256!
+      console.log(`Force exiting with code ${CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT}...`);
+      process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT); // GitHub Actions Docker uses values ranged from 0 to 255. Any bigger value will be % 256
     } else {
+      let waitTime = null;
+
+      if (retryAfter) {
+        waitTime = isNaN(retryAfter)
+          ? Math.ceil((new Date(retryAfter).getTime() - Date.now()) / 1000) // HTTP date
+          : parseInt(retryAfter, 10); // Seconds
+        console.log(`'retry-after' headers found.`);
+      } else if (rateLimitReset) {
+        waitTime = Math.max(parseInt(rateLimitReset, 10) - Math.floor(Date.now() / 1000), 0);
+        console.log(`'x-ratelimit-reset' headers found.`);
+      } else {
+        waitTime = CONFIG.CRAWLER.CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS / 1000;
+        console.log(`No 'retry-after' or 'x-ratelimit-reset' headers found. Using fallback delay.`);
+      }
+      console.log(`Retrying after ${waitTime} seconds...`);
       await delay(waitTime * 1000);
     }
   }

package/src/utils/crawl/url/processor.js

@@ -7,7 +7,7 @@ import { saveDataset, saveQueue } from '../fileOperations.js';
 
 export const processURL = async (entry, fileNumber, urlData) => {
   const startTime = new Date().getTime();
-  const { url, referrer, depth } = entry;
+  const { url, depth } = entry;
 
   if (entry.file || (entry.error && !(await shouldRetry({ response: { status: entry.status } })))) return;
   
@@ -21,7 +21,7 @@ export const processURL = async (entry, fileNumber, urlData) => {
       enqueueURLs(urlData, $, url, depth + 1);
 
       const content = cleanHTML($);
-      const filename = saveDataset({ url, referrerURL: referrer, statusCode: status, depth, content }, fileNumber);
+      const filename = saveDataset({ url, content }, fileNumber);
 
       entry.file = filename;
       entry.status = status;