scraply 1.0.2 → 1.0.4

package/.github/workflows/npm-publish.yml

@@ -0,0 +1,30 @@
+name: Publish to NPM
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '20.17'
+          registry-url: 'https://registry.npmjs.org/'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Publish to npm
+        run: npm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {
@@ -16,5 +16,9 @@
     "axios": "^1.7.7",
     "cheerio": "^1.0.0",
     "he": "^1.2.0"
+  },
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/",
+    "access": "public"
   }
 }

package/readme.md

@@ -1,4 +1,89 @@
-Scraply is a customizable and efficient web crawler and data scraper for Node.js, designed to handle various web crawling needs with ease. You can define the URLs to crawl, configure patterns to include/exclude, and format the output data in JSON. Scraply is built to be flexible, with user-configurable settings and dynamic paths.
-
-Installation
-`npm install scraply`
+# Scraply
+Scraply is a customizable and efficient web crawler and data scraper for Node.js, designed to handle various web crawling needs with ease. You can define the URLs to crawl, configure patterns to include/exclude, and format the output data in JSON. Scraply is built to be flexible, with user-configurable settings and dynamic paths.
+
+Bug Reports & Dev Stuff on: [Scraply's GitHub](https://github.com/pauserratgutierrez/scraply)
+
+## Installation
+Using npm:
+``npm install scraply``
+
+## Working Example
+Initialize Scraply with provided URLs to start crawling:
+```
+import { scraply } from 'scraply';
+scraply({
+  CRAWLER: {
+    INITIAL_URLS: ['https://example.com']
+  }
+});
+```
+
+## How Scraply Works
+### Persistent Data Storage
+Scraply persistently saves the state of the crawler in JSON files (the queue, crawled data, etc.). If the crawler is interrupted or rate-limited, all progress is saved, and the crawler will automatically stop. When restarted, Scraply resumes crawling exactly where it left off, without reprocessing already crawled URLs.
+
+### Handling Rate Limiting
+Scraply is designed to handle rate-limiting gracefully. If the crawler encounters rate-limited responses (e.g., status code `429`), it stops processing further requests and saves everything in the queue. Once restarted, it resumes the crawling process from where it stopped.
+
+This makes Scraply ideal for long-running, continuous crawling tasks. You can integrate Scraply with GitHub Actions or other CI/CD pipelines to perform endless crawling jobs over time. Simply schedule Scraply to run periodically, and it will continue gathering data without duplicating work.
+
+### Integration with GitHub Actions
+Scraply can be easily integrated into a GitHub Action workflow for continuous, long-running crawling tasks. You can set it up to crawl for a set duration or number of URLs, persistently saving the progress, and then resuming where it left off on the next run.
+
+## Config Options
+Scraply allows you to pass a configuration object to the main ```scraply()``` function to customize the crawling behavior. Below are the current configuration options:
+```
+MAIN_DIR: 'dataset',
+
+CRAWLER: {
+  INITIAL_URLS: [
+    'https://crawler-test.com/'
+  ],
+  INCLUDE_URLS: [
+    'https://crawler-test.com/.*'
+  ],
+  ALLOWED_CONTENT_TYPES: [
+    'text/html'
+  ],
+  EXCLUDE_PATTERNS: [
+    '/cdn-cgi/',
+    /\.(zip|rar|webp|png|jpg|jpeg|gif|mp3|mp4|pdf|css|js|svg|ico|eot|ttf|woff|woff2|otf|webm|ogg|wav|flac|m4a|mkv|mov|avi|wmv|flv|swf|exe|msi|dmg|iso|bin)$/,
+  ],
+  DOM_ELEMENTS_REMOVE: [
+    'script',
+    'noscript',
+    'style',
+    'meta',
+    'link',
+    'svg',
+    'path',
+    'img',
+    'input',
+    'textarea',
+    'embed',
+    'object',
+    'iframe',
+    'nav',
+    'header',
+    'footer',
+    'aside',
+    'button'
+  ],
+  RETRY_STATUS_CODES: [408, 429, 500, 502, 503, 504],
+  REQUEST_TIMEOUT: 4000,
+  MAX_REDIRECTS: 3,
+  MAX_RETRIES: 2,
+  CRAWL_DELAY_MS: 200,
+  CRAWL_ERROR_RETRY_DELAY_MS: 800,
+},
+
+DATA_FORMATTER: {
+  EXCLUDED_PATTERNS: [],
+  CATEGORISED_PATHS: {
+    'https://crawler-test.com': {
+      'mobile': 'mobile.json',
+      'fallback': 'general.json'
+    },
+  },
+}
+```

package/src/{config.js → defaultConfig.js}

@@ -1,55 +1,55 @@
-export const DEFAULT_CONFIG = {
-  MAIN_DIR: 'dataset',
-
-  CRAWLER: {
-    INITIAL_URLS: [
-      'https://crawler-test.com/'
-    ],
-    INCLUDE_URLS: [
-      'https://crawler-test.com/.*'
-    ],
-    ALLOWED_CONTENT_TYPES: [
-      'text/html'
-    ],
-    EXCLUDE_PATTERNS: [
-      '/cdn-cgi/',
-      /\.(zip|rar|webp|png|jpg|jpeg|gif|mp3|mp4|pdf|css|js|svg|ico|eot|ttf|woff|woff2|otf|webm|ogg|wav|flac|m4a|mkv|mov|avi|wmv|flv|swf|exe|msi|dmg|iso|bin)$/,
-    ],
-    DOM_ELEMENTS_REMOVE: [
-      'script',
-      'noscript',
-      'style',
-      'meta',
-      'link',
-      'svg',
-      'path',
-      'img',
-      'input',
-      'textarea',
-      'embed',
-      'object',
-      'iframe',
-      'nav',
-      'header',
-      'footer',
-      'aside',
-      'button'
-    ],
-    RETRY_STATUS_CODES: [408, 429, 500, 502, 503, 504],
-    REQUEST_TIMEOUT: 4000,
-    MAX_REDIRECTS: 3,
-    MAX_RETRIES: 2,
-    CRAWL_DELAY_MS: 200,
-    CRAWL_ERROR_RETRY_DELAY_MS: 800,
-  },
-
-  DATA_FORMATTER: {
-    EXCLUDED_PATTERNS: [],
-    CATEGORISED_PATHS: {
-      'https://crawler-test.com': {
-        'mobile': 'mobile.json',
-        'fallback': 'general.json'
-      },
-    },
-  }
-};
+export const DEFAULT_CONFIG = {
+  MAIN_DIR: 'dataset',
+
+  CRAWLER: {
+    INITIAL_URLS: [
+      'https://crawler-test.com/'
+    ],
+    INCLUDE_URLS: [
+      'https://crawler-test.com/.*'
+    ],
+    ALLOWED_CONTENT_TYPES: [
+      'text/html'
+    ],
+    EXCLUDE_PATTERNS: [
+      '/cdn-cgi/',
+      /\.(zip|rar|webp|png|jpg|jpeg|gif|mp3|mp4|pdf|css|js|svg|ico|eot|ttf|woff|woff2|otf|webm|ogg|wav|flac|m4a|mkv|mov|avi|wmv|flv|swf|exe|msi|dmg|iso|bin)$/,
+    ],
+    DOM_ELEMENTS_REMOVE: [
+      'script',
+      'noscript',
+      'style',
+      'meta',
+      'link',
+      'svg',
+      'path',
+      'img',
+      'input',
+      'textarea',
+      'embed',
+      'object',
+      'iframe',
+      'nav',
+      'header',
+      'footer',
+      'aside',
+      'button'
+    ],
+    RETRY_STATUS_CODES: [408, 429, 500, 502, 503, 504],
+    REQUEST_TIMEOUT: 4000,
+    MAX_REDIRECTS: 3,
+    MAX_RETRIES: 2,
+    CRAWL_DELAY_MS: 200,
+    CRAWL_ERROR_RETRY_DELAY_MS: 800,
+  },
+
+  DATA_FORMATTER: {
+    EXCLUDED_PATTERNS: [],
+    CATEGORISED_PATHS: {
+      'https://crawler-test.com': {
+        'mobile': 'mobile.json',
+        'fallback': 'general.json'
+      },
+    },
+  }
+};

package/src/loadConfig.js

@@ -1,25 +1,30 @@
-import path from 'node:path';
-import { DEFAULT_CONFIG } from './config.js';
-
-// A utility function to perform a deep merge of objects
-function deepMerge(target, source) {
-  for (const key in source) {
-    if (source[key] instanceof Object && key in target) {
-      Object.assign(source[key], deepMerge(target[key], source[key]));
-    }
-  }
-  return { ...target, ...source };
-};
-
-export function loadConfig(userConfig = {}) {
-  // Merge the user config with the default config
-  const config = deepMerge(DEFAULT_CONFIG, userConfig);
-
-  // Dynamically construct paths using MAIN_DIR
-  config.CRAWLER.QUEUE_PATH = path.join(config.MAIN_DIR, 'queue.json');
-  config.CRAWLER.CRAWLED_PATH = path.join(config.MAIN_DIR, 'crawled');
-  config.DATA_FORMATTER.FORMATTED_PATH = path.join(config.MAIN_DIR, 'formatted');
-  config.DATA_FORMATTER.ERROR_REPORT_PATH = path.join(config.MAIN_DIR, 'error-report.json');
-
-  return config;
-};
+import path from 'node:path';
+import { DEFAULT_CONFIG } from './defaultConfig.js';
+
+// A utility function to perform a deep merge of objects
+function deepMerge(target, source) {
+  for (const key in source) {
+    if (source[key] instanceof Object && key in target) {
+      Object.assign(source[key], deepMerge(target[key], source[key]));
+    }
+  }
+  return { ...target, ...source };
+};
+
+export function loadConfig(userConfig = {}) {
+  // Merge the user config with the default config
+  const config = deepMerge(DEFAULT_CONFIG, userConfig);
+
+  // Dynamically construct paths using MAIN_DIR
+  config.CRAWLER.QUEUE_PATH = path.join(config.MAIN_DIR, 'queue.json');
+  config.CRAWLER.CRAWLED_PATH = path.join(config.MAIN_DIR, 'crawled');
+  config.DATA_FORMATTER.FORMATTED_PATH = path.join(config.MAIN_DIR, 'formatted');
+  config.DATA_FORMATTER.ERROR_REPORT_PATH = path.join(config.MAIN_DIR, 'error-report.json');
+
+  // If INCLUDE_URLS is not specified, set it to INITIAL_URLS by default
+  if (!config.CRAWLER.INCLUDE_URLS || config.CRAWLER.INCLUDE_URLS.length === 0) {
+    config.CRAWLER.INCLUDE_URLS = config.CRAWLER.INITIAL_URLS.map(url => `${url}.*`);
+  }
+
+  return config;
+};

package/src/scraply.js

@@ -1,121 +1,121 @@
-import { loadConfig } from './loadConfig.js';
-import { normalizeURL } from './utils/crawl/url/normalize.js';
-import { loadJSON, saveQueue, deleteDataFiles } from './utils/crawl/fileOperations.js';
-import { processURL } from './utils/crawl/url/processor.js';
-import { formatData, saveSortedFormattedJSON, saveHardcodedExtraLinks } from './utils/format/formatData.js';
-
-let urlData = [];
-let urlMetadata = {};
-let CONFIG = {};
-
-const initializeCrawler = () => {
-  urlData = loadJSON(CONFIG.CRAWLER.QUEUE_PATH);
-
-  if (urlData.length === 0) { // If the queue is empty, start fresh with the initial URLs.
-    console.log(`Starting fresh! No URLs found in ${CONFIG.CRAWLER.QUEUE_PATH}\n`);
-  
-    CONFIG.CRAWLER.INITIAL_URLS.forEach(url => {
-      const normalizedURL = normalizeURL(url);
-      urlData.push({ url: normalizedURL, file: null, status: null, error: null });
-    });
-    saveQueue(urlData);
-  } else { // If the queue is not empty
-    const allProcessed = urlData.every(entry => entry.file !== null || entry.error !== null);
-    if (allProcessed) { // If all URLs have been processed
-      console.log(`All URLs in ${CONFIG.CRAWLER.QUEUE_PATH} have been processed. Deleting persistent storage and starting a fresh Crawl...\n`);
-
-      urlData = [];
-      urlMetadata = {};
-
-      // Delete everything except CONFIG.DATA_FORMATTER.FORMATTED_PATH, so that the formatted data is always preserved until the crawler really finalizes the data. This way, the Discord Bot will fetch the correct & latest data from the GitHub repo, without fetching any incomplete data or empty data, as it watches for file diffs!
-      deleteDataFiles(CONFIG.CRAWLER.QUEUE_PATH);
-      deleteDataFiles(CONFIG.CRAWLER.CRAWLED_PATH);
-      deleteDataFiles(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH);
-
-      initializeCrawler();
-    } else { // If there are URLs that haven't been processed yet, resume from the queue.
-      console.log(`Resuming from ${CONFIG.CRAWLER.QUEUE_PATH} with ${urlData.length} total found URLs\n`);
-    }
-  }
-};
-
-const crawler = async () => {
-  console.log(`STARTING CRAWLER
-  - Initial URLs: ${CONFIG.CRAWLER.INITIAL_URLS}
-  - Include URLs: ${CONFIG.CRAWLER.INCLUDE_URLS}
-  - Excluded Patterns: ${CONFIG.CRAWLER.EXCLUDE_PATTERNS}
-  - Allowed Content Types: ${CONFIG.CRAWLER.ALLOWED_CONTENT_TYPES}
-  - Retry Status Codes: ${CONFIG.CRAWLER.RETRY_STATUS_CODES}
-  - Request Timeout: ${CONFIG.CRAWLER.REQUEST_TIMEOUT}
-  - Max Redirects: ${CONFIG.CRAWLER.MAX_REDIRECTS}
-  - Max Retries: ${CONFIG.CRAWLER.MAX_RETRIES}
-  - Crawl Delay: ${CONFIG.CRAWLER.CRAWL_DELAY_MS}ms
-  - Crawl Error Retry Delay: ${CONFIG.CRAWLER.CRAWL_ERROR_RETRY_DELAY_MS}ms
-  `);
-
-  let fileNumber = urlData.filter(entry => entry.file).length + 1;
-  for await (const entry of urlData) {
-    if (!entry.file) { // Only process URLs that haven't been processed yet.
-      await processURL(entry, fileNumber, urlData, urlMetadata);
-      fileNumber++; // Increment the file number only if the URL was processed successfully.
-    }
-  }
-
-  const totalUrls = urlData.length;
-  const crawledUrls = urlData.filter(entry => entry.file !== null).length;
-  const notCrawledUrls = totalUrls - crawledUrls;
-  const errorUrls = urlData.filter(entry => entry.error !== null);
-
-  console.log(`\nCRAWLING COMPLETED! ${crawledUrls} of ${totalUrls} (${notCrawledUrls} not crawled)`);
-
-  // Iterate over all the urlData and save all the url & content to files, categorized by CONFIG.DATA_FORMATTER.CATEGORISED_PATHS. Exclude the URLs that match the patterns in CONFIG.DATA_FORMATTER.EXCLUDED_PATTERNS. Save in CONFIG.DATA_FORMATTER.FORMATTED_PATH.
-  console.log(`\nFORMATTING DATA...`);
-  const dataToSave = {};
-
-  for await (const entry of urlData) {
-    const savePath = formatData(entry);
-    if (savePath) { // If the URL should be saved.
-      if (!dataToSave[savePath]) dataToSave[savePath] = [];
-
-      // Load content from the file referenced by entry.file
-      let content = null;
-      try {
-        content = loadJSON(entry.file);
-        dataToSave[savePath].push({ url: entry.url, content: content.content });
-      } catch (e) {
-        console.error(`Error loading content from ${entry.file}: ${e.message}`);
-      }
-    }
-  };
-
-  // Save the data to files.
-  let totalSavedURLs = 0;
-  for (const [savePath, data] of Object.entries(dataToSave)) {
-    totalSavedURLs += data.length;
-    console.log(`${data.length} -> ${savePath}`);
-    saveSortedFormattedJSON(savePath, data);
-  };
-  console.log(`${totalSavedURLs} total saved URLs to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
-
-  // Save hardcoded extra links to files.
-  await saveHardcodedExtraLinks();
-  console.log(`Hardcoded extra links saved to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
-
-  // Error reporting: Save into CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH the URLs that had any error: Save the url, the referrer, status code and error!
-  const errorData = errorUrls.map(entry => {
-    return { url: entry.url, status: entry.status, error: entry.error };
-  });
-
-  saveSortedFormattedJSON(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH, errorData);
-
-  console.log(`Errors: ${errorData.length} -> ${CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH}.`);
-};
-
-// Main function to be exported and used
-export const scraply = async (userConfig = {}) => {
-  CONFIG = loadConfig(userConfig);
-  global.CONFIG = CONFIG;
-
-  initializeCrawler();
-  await crawler();
-};
+import { loadConfig } from './loadConfig.js';
+import { normalizeURL } from './utils/crawl/url/normalize.js';
+import { loadJSON, saveQueue, deleteDataFiles } from './utils/crawl/fileOperations.js';
+import { processURL } from './utils/crawl/url/processor.js';
+import { formatData, saveSortedFormattedJSON, saveHardcodedExtraLinks } from './utils/format/formatData.js';
+
+let urlData = [];
+let urlMetadata = {};
+let CONFIG = {};
+
+const init = () => {
+  urlData = loadJSON(CONFIG.CRAWLER.QUEUE_PATH);
+
+  if (urlData.length === 0) { // If the queue is empty, start fresh with the initial URLs.
+    console.log(`Starting fresh! No URLs found in ${CONFIG.CRAWLER.QUEUE_PATH}\n`);
+  
+    CONFIG.CRAWLER.INITIAL_URLS.forEach(url => {
+      const normalizedURL = normalizeURL(url);
+      urlData.push({ url: normalizedURL, file: null, status: null, error: null });
+    });
+    saveQueue(urlData);
+  } else { // If the queue is not empty
+    const allProcessed = urlData.every(entry => entry.file !== null || entry.error !== null);
+    if (allProcessed) { // If all URLs have been processed
+      console.log(`All URLs in ${CONFIG.CRAWLER.QUEUE_PATH} have been processed. Deleting persistent storage and starting a fresh Crawl...\n`);
+
+      urlData = [];
+      urlMetadata = {};
+
+      // Delete everything except CONFIG.DATA_FORMATTER.FORMATTED_PATH, so that the formatted data is always preserved until the crawler really finalizes the data. This way, the Discord Bot will fetch the correct & latest data from the GitHub repo, without fetching any incomplete data or empty data, as it watches for file diffs!
+      deleteDataFiles(CONFIG.CRAWLER.QUEUE_PATH);
+      deleteDataFiles(CONFIG.CRAWLER.CRAWLED_PATH);
+      deleteDataFiles(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH);
+
+      init();
+    } else { // If there are URLs that haven't been processed yet, resume from the queue.
+      console.log(`Resuming from ${CONFIG.CRAWLER.QUEUE_PATH} with ${urlData.length} total found URLs\n`);
+    }
+  }
+};
+
+const start = async () => {
+  console.log(`STARTING SCRAPLY CRAWLER...
+  - Initial URLs: ${CONFIG.CRAWLER.INITIAL_URLS}
+  - Include URLs: ${CONFIG.CRAWLER.INCLUDE_URLS}
+  - Excluded Patterns: ${CONFIG.CRAWLER.EXCLUDE_PATTERNS}
+  - Allowed Content Types: ${CONFIG.CRAWLER.ALLOWED_CONTENT_TYPES}
+  - Retry Status Codes: ${CONFIG.CRAWLER.RETRY_STATUS_CODES}
+  - Request Timeout: ${CONFIG.CRAWLER.REQUEST_TIMEOUT}
+  - Max Redirects: ${CONFIG.CRAWLER.MAX_REDIRECTS}
+  - Max Retries: ${CONFIG.CRAWLER.MAX_RETRIES}
+  - Crawl Delay: ${CONFIG.CRAWLER.CRAWL_DELAY_MS}ms
+  - Crawl Error Retry Delay: ${CONFIG.CRAWLER.CRAWL_ERROR_RETRY_DELAY_MS}ms
+  `);
+
+  let fileNumber = urlData.filter(entry => entry.file).length + 1;
+  for await (const entry of urlData) {
+    if (!entry.file) { // Only process URLs that haven't been processed yet.
+      await processURL(entry, fileNumber, urlData, urlMetadata);
+      fileNumber++; // Increment the file number only if the URL was processed successfully.
+    }
+  }
+
+  const totalUrls = urlData.length;
+  const crawledUrls = urlData.filter(entry => entry.file !== null).length;
+  const notCrawledUrls = totalUrls - crawledUrls;
+  const errorUrls = urlData.filter(entry => entry.error !== null);
+
+  console.log(`\nCRAWLING COMPLETED! ${crawledUrls} of ${totalUrls} (${notCrawledUrls} not crawled)`);
+
+  // Iterate over all the urlData and save all the url & content to files, categorized by CONFIG.DATA_FORMATTER.CATEGORISED_PATHS. Exclude the URLs that match the patterns in CONFIG.DATA_FORMATTER.EXCLUDED_PATTERNS. Save in CONFIG.DATA_FORMATTER.FORMATTED_PATH.
+  console.log(`\nFORMATTING DATA...`);
+  const dataToSave = {};
+
+  for await (const entry of urlData) {
+    const savePath = formatData(entry);
+    if (savePath) { // If the URL should be saved.
+      if (!dataToSave[savePath]) dataToSave[savePath] = [];
+
+      // Load content from the file referenced by entry.file
+      let content = null;
+      try {
+        content = loadJSON(entry.file);
+        dataToSave[savePath].push({ url: entry.url, content: content.content });
+      } catch (e) {
+        console.error(`Error loading content from ${entry.file}: ${e.message}`);
+      }
+    }
+  };
+
+  // Save the data to files.
+  let totalSavedURLs = 0;
+  for (const [savePath, data] of Object.entries(dataToSave)) {
+    totalSavedURLs += data.length;
+    console.log(`${data.length} -> ${savePath}`);
+    saveSortedFormattedJSON(savePath, data);
+  };
+  console.log(`${totalSavedURLs} total saved URLs to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
+
+  // Save hardcoded extra links to files.
+  await saveHardcodedExtraLinks();
+  console.log(`Hardcoded extra links saved to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
+
+  // Error reporting: Save into CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH the URLs that had any error: Save the url, the referrer, status code and error!
+  const errorData = errorUrls.map(entry => {
+    return { url: entry.url, status: entry.status, error: entry.error };
+  });
+
+  saveSortedFormattedJSON(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH, errorData);
+
+  console.log(`Errors: ${errorData.length} -> ${CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH}.`);
+};
+
+// Main function to be exported and used
+export const scraply = async (userConfig = {}) => {
+  CONFIG = loadConfig(userConfig);
+  global.CONFIG = CONFIG;
+
+  init();
+  await start();
+};

package/src/utils/crawl/cleanHTML.js

@@ -1,38 +1,38 @@
-import he from 'he';
-
-export const cleanHTML = ($) => {
-  // Remove unwanted elements
-  const $aux = $;
-  $aux(CONFIG.CRAWLER.DOM_ELEMENTS_REMOVE.join(',')).remove();
-  $aux('*').contents().filter((_, el) => el.type === 'comment').remove();
-
-  // Get the text content of the body and decode HTML entities
-  // let bodyText = he.decode($aux('body').text(), { level: 'all' });
-
-  // Get the text content of the body element, ensuring spaces between child elements
-  let bodyText = getTextWithSpaces($aux, $aux('body'));
-
-  // Decode HTML entities
-  bodyText = he.decode(bodyText, { level: 'all' });
-
-  // Clean up the resulting text
-  return bodyText
-    .replace(/\n/g, ' ') // Replace newlines with a space
-    .replace(/\\['"\\]/g, match => match.slice(1)) // Replace escaped characters with the unescaped character
-    .replace(/[\u200B\u00A0\u2028\u2029\u202F\u00AD\u2060\uFEFF]/g, ' ') // Replace zero-width spaces with a space
-    .replace(/\s{2,}/g, ' ') // Replace multiple spaces with a single space
-    .trim();
-};
-
-// Custom function to get text content with spaces between elements
-const getTextWithSpaces = ($, element) => {
-  let text = '';
-  element.contents().each((_, el) => {
-    if (el.type === 'text') {
-      text += $(el).text() + ' ';
-    } else if (el.type === 'tag') {
-      text += getTextWithSpaces($, $(el));
-    }
-  });
-  return text;
-};
+import he from 'he';
+
+export const cleanHTML = ($) => {
+  // Remove unwanted elements
+  const $aux = $;
+  $aux(CONFIG.CRAWLER.DOM_ELEMENTS_REMOVE.join(',')).remove();
+  $aux('*').contents().filter((_, el) => el.type === 'comment').remove();
+
+  // Get the text content of the body and decode HTML entities
+  // let bodyText = he.decode($aux('body').text(), { level: 'all' });
+
+  // Get the text content of the body element, ensuring spaces between child elements
+  let bodyText = getTextWithSpaces($aux, $aux('body'));
+
+  // Decode HTML entities
+  bodyText = he.decode(bodyText, { level: 'all' });
+
+  // Clean up the resulting text
+  return bodyText
+    .replace(/\n/g, ' ') // Replace newlines with a space
+    .replace(/\\['"\\]/g, match => match.slice(1)) // Replace escaped characters with the unescaped character
+    .replace(/[\u200B\u00A0\u2028\u2029\u202F\u00AD\u2060\uFEFF]/g, ' ') // Replace zero-width spaces with a space
+    .replace(/\s{2,}/g, ' ') // Replace multiple spaces with a single space
+    .trim();
+};
+
+// Custom function to get text content with spaces between elements
+const getTextWithSpaces = ($, element) => {
+  let text = '';
+  element.contents().each((_, el) => {
+    if (el.type === 'text') {
+      text += $(el).text() + ' ';
+    } else if (el.type === 'tag') {
+      text += getTextWithSpaces($, $(el));
+    }
+  });
+  return text;
+};

package/src/utils/crawl/delay.js

@@ -1 +1 @@
-export const delay = async (ms) => new Promise(resolve => setTimeout(resolve, ms));
+export const delay = async (ms) => new Promise(resolve => setTimeout(resolve, ms));

package/src/utils/crawl/fileOperations.js

@@ -1,37 +1,37 @@
-import fs from 'node:fs';
-import path from 'node:path';
-
-export const loadJSON = (filePath) => fs.existsSync(filePath) ? JSON.parse(fs.readFileSync(filePath, 'utf8')) : [];
-
-export const saveJSON = (filePath, data) => {
-  const dir = path.dirname(filePath);
-  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
-  return fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8');
-};
-
-export const saveDataset = (data, fileNumber) => {
-  if (!fs.existsSync(CONFIG.CRAWLER.CRAWLED_PATH)) fs.mkdirSync(CONFIG.CRAWLER.CRAWLED_PATH, { recursive: true });
-  const filename = `${CONFIG.CRAWLER.CRAWLED_PATH}/${fileNumber}.json`;
-  fs.writeFileSync(filename, JSON.stringify(data, null, 2), 'utf8');
-  return filename;
-};
-
-export const saveQueue = (urlData) => saveJSON(CONFIG.CRAWLER.QUEUE_PATH, urlData);
-
-export const deleteDataFiles = (filePath) => {
-  if (fs.existsSync(filePath)) {
-    if (fs.lstatSync(filePath).isDirectory()) {
-      fs.readdirSync(filePath).forEach((file) => {
-        const currentPath = path.join(filePath, file);
-        if (fs.lstatSync(currentPath).isDirectory()) {
-          deleteDataFiles(currentPath);
-        } else {
-          fs.unlinkSync(currentPath);
-        }
-      });
-      fs.rmdirSync(filePath);
-    } else {
-      fs.unlinkSync(filePath);
-    }
-  }
-};
+import fs from 'node:fs';
+import path from 'node:path';
+
+export const loadJSON = (filePath) => fs.existsSync(filePath) ? JSON.parse(fs.readFileSync(filePath, 'utf8')) : [];
+
+export const saveJSON = (filePath, data) => {
+  const dir = path.dirname(filePath);
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+  return fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8');
+};
+
+export const saveDataset = (data, fileNumber) => {
+  if (!fs.existsSync(CONFIG.CRAWLER.CRAWLED_PATH)) fs.mkdirSync(CONFIG.CRAWLER.CRAWLED_PATH, { recursive: true });
+  const filename = `${CONFIG.CRAWLER.CRAWLED_PATH}/${fileNumber}.json`;
+  fs.writeFileSync(filename, JSON.stringify(data, null, 2), 'utf8');
+  return filename;
+};
+
+export const saveQueue = (urlData) => saveJSON(CONFIG.CRAWLER.QUEUE_PATH, urlData);
+
+export const deleteDataFiles = (filePath) => {
+  if (fs.existsSync(filePath)) {
+    if (fs.lstatSync(filePath).isDirectory()) {
+      fs.readdirSync(filePath).forEach((file) => {
+        const currentPath = path.join(filePath, file);
+        if (fs.lstatSync(currentPath).isDirectory()) {
+          deleteDataFiles(currentPath);
+        } else {
+          fs.unlinkSync(currentPath);
+        }
+      });
+      fs.rmdirSync(filePath);
+    } else {
+      fs.unlinkSync(filePath);
+    }
+  }
+};

package/src/utils/crawl/url/fetch.js

@@ -1,25 +1,25 @@
-import axios from 'axios';
-import { delay } from '../delay.js';
-import { shouldRetry } from './handlers.js';
-
-export const fetchURL = async (url, retries = 2) => {
-  try {
-    const response = await axios.get(url, { timeout: CONFIG.CRAWLER.REQUEST_TIMEOUT, maxRedirects: CONFIG.CRAWLER.MAX_REDIRECTS });
-    const contentType = response.headers['content-type'];
-
-    if (!CONFIG.CRAWLER.ALLOWED_CONTENT_TYPES.some(type => contentType.includes(type))) {
-      return { error: `Content-Type ${contentType} is not allowed.`, status: response.status };
-    };
-
-    return { data: response.data, status: response.status };
-  } catch (error) {
-    if (retries > 0 && shouldRetry(error)) {
-      console.log(`Retrying (${CONFIG.CRAWLER.MAX_RETRIES - retries + 1}/${CONFIG.CRAWLER.MAX_RETRIES}) -> ${url}`);
-      if (CONFIG.CRAWLER.CRAWL_ERROR_RETRY_DELAY_MS > 0) await delay(CONFIG.CRAWLER.CRAWL_ERROR_RETRY_DELAY_MS);
-      return fetchURL(url, retries - 1);
-    } else {
-      console.error(`Failed to fetch ${url} -> ${error.message}`);
-      return { error: error.message, status: error.response ? error.response.status : null };
-    };
-  };
-};
+import axios from 'axios';
+import { delay } from '../delay.js';
+import { shouldRetry } from './handlers.js';
+
+export const fetchURL = async (url, retries = 2) => {
+  try {
+    const response = await axios.get(url, { timeout: CONFIG.CRAWLER.REQUEST_TIMEOUT, maxRedirects: CONFIG.CRAWLER.MAX_REDIRECTS });
+    const contentType = response.headers['content-type'];
+
+    if (!CONFIG.CRAWLER.ALLOWED_CONTENT_TYPES.some(type => contentType.includes(type))) {
+      return { error: `Content-Type ${contentType} is not allowed.`, status: response.status };
+    };
+
+    return { data: response.data, status: response.status };
+  } catch (error) {
+    if (retries > 0 && shouldRetry(error)) {
+      console.log(`Retrying (${CONFIG.CRAWLER.MAX_RETRIES - retries + 1}/${CONFIG.CRAWLER.MAX_RETRIES}) -> ${url}`);
+      if (CONFIG.CRAWLER.CRAWL_ERROR_RETRY_DELAY_MS > 0) await delay(CONFIG.CRAWLER.CRAWL_ERROR_RETRY_DELAY_MS);
+      return fetchURL(url, retries - 1);
+    } else {
+      console.error(`Failed to fetch ${url} -> ${error.message}`);
+      return { error: error.message, status: error.response ? error.response.status : null };
+    };
+  };
+};

package/src/utils/crawl/url/handlers.js

@@ -1,44 +1,44 @@
-import { URL } from 'node:url';
-import { normalizeURL } from './normalize.js';
-
-// Handle HTML Status Codes HERE!
-export const shouldRetry = (error) => {
-  if (!error.response) return true;
-  if (error.response.status === 429) {
-    const waitTime = error.response.headers ? error.response.headers['retry-after'] : null;
-    if (waitTime) {
-      console.log(`Rate limited for ${waitTime} seconds, exiting Crawler...`);
-    } else {
-      console.log(`Rate limited, no retry-after header found, exiting Crawler...`);
-    }
-    process.exit(10); // GitHub Actions Docker uses values ranged from 0 to 255, so any bigger value will be % 256!
-  }
-  return CONFIG.CRAWLER.RETRY_STATUS_CODES.includes(error.response.status); // Retry only on specific status codes
-};
-
-const shouldIncludeURL = (url) => {
-  try {
-    const urlObj = new URL(url);
-    return CONFIG.CRAWLER.INCLUDE_URLS.some(pattern => new RegExp(pattern).test(urlObj.toString())) && !CONFIG.CRAWLER.EXCLUDE_PATTERNS.some(pattern => new RegExp(pattern).test(urlObj.pathname));
-  } catch (error) {
-    return false;
-  }
-};
-
-export const enqueueURLs = (urlData, urlMetadata, $, baseURL, referrer, depth) => {
-  $('a[href]').each((_, element) => {
-    const href = $(element).attr('href');
-    if (!href) return;
-
-    try {
-      const newURL = new URL(href, baseURL).toString();
-      const normalizedURL = normalizeURL(newURL);
-      if (shouldIncludeURL(normalizedURL) && !urlData.some(entry => entry.url === normalizedURL)) {
-        urlData.push({ url: normalizedURL, file: null, status: null, error: null });
-        urlMetadata[normalizedURL] = { referrer, depth };
-      }
-    } catch (error) {
-      console.error(`Failed to enqueue URL: ${href} from ${baseURL}: ${error.message}`);
-    }
-  });
-};
+import { URL } from 'node:url';
+import { normalizeURL } from './normalize.js';
+
+// Handle HTML Status Codes HERE!
+export const shouldRetry = (error) => {
+  if (!error.response) return true;
+  if (error.response.status === 429) {
+    const waitTime = error.response.headers ? error.response.headers['retry-after'] : null;
+    if (waitTime) {
+      console.log(`Rate limited for ${waitTime} seconds, exiting Crawler...`);
+    } else {
+      console.log(`Rate limited, no retry-after header found, exiting Crawler...`);
+    }
+    process.exit(10); // GitHub Actions Docker uses values ranged from 0 to 255, so any bigger value will be % 256!
+  }
+  return CONFIG.CRAWLER.RETRY_STATUS_CODES.includes(error.response.status); // Retry only on specific status codes
+};
+
+const shouldIncludeURL = (url) => {
+  try {
+    const urlObj = new URL(url);
+    return CONFIG.CRAWLER.INCLUDE_URLS.some(pattern => new RegExp(pattern).test(urlObj.toString())) && !CONFIG.CRAWLER.EXCLUDE_PATTERNS.some(pattern => new RegExp(pattern).test(urlObj.pathname));
+  } catch (error) {
+    return false;
+  }
+};
+
+export const enqueueURLs = (urlData, urlMetadata, $, baseURL, referrer, depth) => {
+  $('a[href]').each((_, element) => {
+    const href = $(element).attr('href');
+    if (!href) return;
+
+    try {
+      const newURL = new URL(href, baseURL).toString();
+      const normalizedURL = normalizeURL(newURL);
+      if (shouldIncludeURL(normalizedURL) && !urlData.some(entry => entry.url === normalizedURL)) {
+        urlData.push({ url: normalizedURL, file: null, status: null, error: null });
+        urlMetadata[normalizedURL] = { referrer, depth };
+      }
+    } catch (error) {
+      console.error(`Failed to enqueue URL: ${href} from ${baseURL}: ${error.message}`);
+    }
+  });
+};

package/src/utils/crawl/url/normalize.js

@@ -1,8 +1,8 @@
-export const normalizeURL = (url) => {
-  const urlObj = new URL(url);
-  urlObj.hash = ''; // Remove the fragment part
-  urlObj.search = ''; // Remove the query part
-  urlObj.pathname = urlObj.pathname.endsWith('/') ? urlObj.pathname.slice(0, -1) : urlObj.pathname; // Remove trailing slashes
-  urlObj.pathname = urlObj.pathname === '' ? '/' : urlObj.pathname; // Handle the root URL separately
-  return urlObj.toString();
-};
+export const normalizeURL = (url) => {
+  const urlObj = new URL(url);
+  urlObj.hash = ''; // Remove the fragment part
+  urlObj.search = ''; // Remove the query part
+  urlObj.pathname = urlObj.pathname.endsWith('/') ? urlObj.pathname.slice(0, -1) : urlObj.pathname; // Remove trailing slashes
+  urlObj.pathname = urlObj.pathname === '' ? '/' : urlObj.pathname; // Handle the root URL separately
+  return urlObj.toString();
+};

package/src/utils/crawl/url/processor.js

@@ -1,44 +1,44 @@
-import { delay } from '../delay.js';
-import { cleanHTML } from '../cleanHTML.js';
-
-import * as cheerio from 'cheerio';
-import { shouldRetry, enqueueURLs } from './handlers.js';
-import { fetchURL } from './fetch.js';
-import { saveDataset, saveQueue } from '../fileOperations.js';
-
-export const processURL = async (entry, fileNumber, urlData, urlMetadata) => {
-  if (entry.file || (entry.error && !shouldRetry({ response: { status: entry.status } }))) return;
-
-  console.log(`- ${fileNumber}/${urlData.length} -> ${entry.url}`);
-
-  const { url } = entry;
-  const { referrer, depth } = urlMetadata[url] || { referrer: null, depth: 0 }; // Default depth is 0.
-  
-  const startTime = new Date().getTime();
-  try {
-    const result = await fetchURL(url, CONFIG.CRAWLER.MAX_RETRIES);
-    if (result && result.data) {
-      const { data: html, status } = result;
-      const $ = cheerio.load(html);
-      enqueueURLs(urlData, urlMetadata, $, url, url, depth + 1);
-      const content = cleanHTML($);
-      const filename = saveDataset({ url, referrerURL: referrer, statusCode: status, depth, content }, fileNumber);
-      entry.file = filename;
-      entry.status = status;
-      entry.error = null;
-    } else {
-      entry.error = result.error;
-      entry.status = result.status;
-    }
-  } catch (error) {
-    entry.error = error.message;
-    entry.status = null;
-  }
-
-  saveQueue(urlData);
-
-  const endTime = new Date().getTime();
-  const elapsedTime = endTime - startTime;
-
-  if (CONFIG.CRAWLER.CRAWL_DELAY_MS > 0 && elapsedTime < CONFIG.CRAWLER.CRAWL_DELAY_MS) await delay(CONFIG.CRAWLER.CRAWL_DELAY_MS - elapsedTime);
-};
+import { delay } from '../delay.js';
+import { cleanHTML } from '../cleanHTML.js';
+
+import * as cheerio from 'cheerio';
+import { shouldRetry, enqueueURLs } from './handlers.js';
+import { fetchURL } from './fetch.js';
+import { saveDataset, saveQueue } from '../fileOperations.js';
+
+export const processURL = async (entry, fileNumber, urlData, urlMetadata) => {
+  if (entry.file || (entry.error && !shouldRetry({ response: { status: entry.status } }))) return;
+
+  console.log(`- ${fileNumber}/${urlData.length} -> ${entry.url}`);
+
+  const { url } = entry;
+  const { referrer, depth } = urlMetadata[url] || { referrer: null, depth: 0 }; // Default depth is 0.
+  
+  const startTime = new Date().getTime();
+  try {
+    const result = await fetchURL(url, CONFIG.CRAWLER.MAX_RETRIES);
+    if (result && result.data) {
+      const { data: html, status } = result;
+      const $ = cheerio.load(html);
+      enqueueURLs(urlData, urlMetadata, $, url, url, depth + 1);
+      const content = cleanHTML($);
+      const filename = saveDataset({ url, referrerURL: referrer, statusCode: status, depth, content }, fileNumber);
+      entry.file = filename;
+      entry.status = status;
+      entry.error = null;
+    } else {
+      entry.error = result.error;
+      entry.status = result.status;
+    }
+  } catch (error) {
+    entry.error = error.message;
+    entry.status = null;
+  }
+
+  saveQueue(urlData);
+
+  const endTime = new Date().getTime();
+  const elapsedTime = endTime - startTime;
+
+  if (CONFIG.CRAWLER.CRAWL_DELAY_MS > 0 && elapsedTime < CONFIG.CRAWLER.CRAWL_DELAY_MS) await delay(CONFIG.CRAWLER.CRAWL_DELAY_MS - elapsedTime);
+};

package/src/utils/format/formatData.js

@@ -1,56 +1,56 @@
-import fs from 'fs';
-import path from 'path';
-
-export const formatData = (entry) => {
-  if (entry.file && entry.error === null) {
-    try {
-      const url = new URL(entry.url);
-      const pathname = url.pathname;
-      const isExcluded = CONFIG.DATA_FORMATTER.EXCLUDED_PATTERNS.some(pattern => new RegExp(pattern).test(entry.url));
-  
-      if (!isExcluded) {
-        const categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.[pathname.split('/')[1]] || CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.fallback;
-        if (categorisedPath) {
-          return path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
-        }
-      }
-    } catch (e) {
-      console.error(`Error formatting data for ${entry.url}: ${e.message}`);
-    }
-  }
-
-  return null;
-};
-
-// Sort data consistently to always save it in the same order between each run, so GitHub doesn't show a diff for the same data.
-function sortData(data, sortKey) {
-  return data.sort((a, b) => {
-    if (a[sortKey] < b[sortKey]) return -1;
-    if (a[sortKey] > b[sortKey]) return 1;
-    return 0;
-  });
-};
-
-export const saveSortedFormattedJSON = (filePath, data) => {
-  const dir = path.dirname(filePath);
-  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
-  const sortedData = sortData(data, 'url'); // ensure data is sorted before saving
-  return fs.writeFileSync(filePath, JSON.stringify(sortedData, null, 2), 'utf8');
-};
-
-export const saveHardcodedExtraLinks = async () => {
-  const data = {
-    file_name: 'cs-links.json',
-    data: [
-        {
-          "url": "https://elemn.to/ai",
-          "content": "🧠 AI - How to save time - This page provides valuable insights on how to leverage AI tools for optimizing workflows and saving time across various tasks."
-        },
-    ],
-  };
-
-  const filePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, data.file_name);
-  saveSortedFormattedJSON(filePath, data.data);
-
-  return data.data.length;
-};
+import fs from 'fs';
+import path from 'path';
+
+export const formatData = (entry) => {
+  if (entry.file && entry.error === null) {
+    try {
+      const url = new URL(entry.url);
+      const pathname = url.pathname;
+      const isExcluded = CONFIG.DATA_FORMATTER.EXCLUDED_PATTERNS.some(pattern => new RegExp(pattern).test(entry.url));
+  
+      if (!isExcluded) {
+        const categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.[pathname.split('/')[1]] || CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.fallback;
+        if (categorisedPath) {
+          return path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
+        }
+      }
+    } catch (e) {
+      console.error(`Error formatting data for ${entry.url}: ${e.message}`);
+    }
+  }
+
+  return null;
+};
+
+// Sort data consistently to always save it in the same order between each run, so GitHub doesn't show a diff for the same data.
+function sortData(data, sortKey) {
+  return data.sort((a, b) => {
+    if (a[sortKey] < b[sortKey]) return -1;
+    if (a[sortKey] > b[sortKey]) return 1;
+    return 0;
+  });
+};
+
+export const saveSortedFormattedJSON = (filePath, data) => {
+  const dir = path.dirname(filePath);
+  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+  const sortedData = sortData(data, 'url'); // ensure data is sorted before saving
+  return fs.writeFileSync(filePath, JSON.stringify(sortedData, null, 2), 'utf8');
+};
+
+export const saveHardcodedExtraLinks = async () => {
+  const data = {
+    file_name: 'cs-links.json',
+    data: [
+        {
+          "url": "https://elemn.to/ai",
+          "content": "🧠 AI - How to save time - This page provides valuable insights on how to leverage AI tools for optimizing workflows and saving time across various tasks."
+        },
+    ],
+  };
+
+  const filePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, data.file_name);
+  saveSortedFormattedJSON(filePath, data.data);
+
+  return data.data.length;
+};