scraply 1.0.16 → 1.0.18

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.16",
+  "version": "1.0.18",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/src/defaultConfig.js

@@ -42,8 +42,8 @@ export const DEFAULT_CONFIG = {
     CRAWL_DELAY_MS: 200,
     CRAWL_ERROR_RETRY_DELAY_MS: 800,
     CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS: 60000,
-    EXIT_CODE_RATE_LIMIT: 10,
-    EXIT_ON_RATE_LIMIT: true
+    EXIT_ON_RATE_LIMIT: true,
+    EXIT_CODE_RATE_LIMIT: 10
   },
 
   DATA_FORMATTER: {

package/src/loadConfig.js

@@ -19,17 +19,11 @@ export function loadConfig(userConfig = {}) {
   config.CRAWLER.QUEUE_PATH = path.posix.join(config.MAIN_DIR, 'queue.json');
   config.CRAWLER.CRAWLED_PATH = path.posix.join(config.MAIN_DIR, 'crawled');
   config.DATA_FORMATTER.FORMATTED_PATH = path.posix.join(config.MAIN_DIR, 'formatted');
-  config.DATA_FORMATTER.ERROR_REPORT_PATH = path.posix.join(config.MAIN_DIR, 'error-report.json');
 
   // If INCLUDE_URLS is not specified, set it to INITIAL_URLS by default
   if (!config.CRAWLER.INCLUDE_URLS || config.CRAWLER.INCLUDE_URLS.length === 0) {
     config.CRAWLER.INCLUDE_URLS = config.CRAWLER.INITIAL_URLS.map(url => `${url}.*`);
   }
 
-  // If HARD_CODED_LINKS is missing, set to an empty array by default
-  if (!config.DATA_FORMATTER.HARD_CODED_LINKS) {
-    config.DATA_FORMATTER.HARD_CODED_LINKS = [];
-  }
-
   return config;
 };

package/src/utils/crawl/cleanHTML.js

@@ -6,9 +6,6 @@ export const cleanHTML = ($) => {
   $aux(CONFIG.CRAWLER.DOM_ELEMENTS_REMOVE.join(',')).remove();
   $aux('*').contents().filter((_, el) => el.type === 'comment').remove();
 
-  // Get the text content of the body and decode HTML entities
-  // let bodyText = he.decode($aux('body').text(), { level: 'all' });
-
   // Get the text content of the body element, ensuring spaces between child elements
   let bodyText = getTextWithSpaces($aux, $aux('body'));
 

package/src/utils/crawl/url/handlers.js

@@ -19,7 +19,7 @@ export const shouldRetry = async (error) => {
         : parseInt(retryAfter, 10); // Seconds
       console.log(`Rate limited. Retrying after ${waitTime} seconds...`);
     } else if (rateLimitReset) {
-      waitTime = Math.max(parseInt(rateLimitReset, 10) - Math.floow(Date.now() / 1000), 0);
+      waitTime = Math.max(parseInt(rateLimitReset, 10) - Math.floor(Date.now() / 1000), 0);
       console.log(`Rate limited. Retrying after ${waitTime} seconds...`);
     } else {
       waitTime = CONFIG.CRAWLER.CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS / 1000;

package/src/utils/crawl/url/processor.js

@@ -7,7 +7,7 @@ import { saveDataset, saveQueue } from '../fileOperations.js';
 
 export const processURL = async (entry, fileNumber, urlData) => {
   const startTime = new Date().getTime();
-  const { url, referrer, depth } = entry;
+  const { url, depth } = entry;
 
   if (entry.file || (entry.error && !(await shouldRetry({ response: { status: entry.status } })))) return;
   
@@ -21,7 +21,7 @@ export const processURL = async (entry, fileNumber, urlData) => {
       enqueueURLs(urlData, $, url, depth + 1);
 
       const content = cleanHTML($);
-      const filename = saveDataset({ url, referrerURL: referrer, statusCode: status, depth, content }, fileNumber);
+      const filename = saveDataset({ url, content }, fileNumber);
 
       entry.file = filename;
       entry.status = status;