scraply 1.0.14 → 1.0.16

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.14",
+  "version": "1.0.16",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/readme.md

@@ -76,6 +76,9 @@ CRAWLER: {
   MAX_RETRIES: 2,
   CRAWL_DELAY_MS: 200,
   CRAWL_ERROR_RETRY_DELAY_MS: 800,
+  CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS: 60000,
+  EXIT_CODE_RATE_LIMIT: 10,
+  EXIT_ON_RATE_LIMIT: true
 },
 
 DATA_FORMATTER: {
@@ -85,17 +88,6 @@ DATA_FORMATTER: {
       'mobile': 'mobile.json',
       '*': 'general.json'
     },
-  },
-  HARD_CODED_LINKS: [
-    {
-      file_name: 'hc-links.json',
-      data: [
-        {
-          "url": "https://custom-link.com",
-          "content": "That's a custom link content, you can add as many as you want."
-        },
-      ]
-    }
-  ]
+  }
 }
 ```

package/src/defaultConfig.js

@@ -41,6 +41,9 @@ export const DEFAULT_CONFIG = {
     MAX_RETRIES: 2,
     CRAWL_DELAY_MS: 200,
     CRAWL_ERROR_RETRY_DELAY_MS: 800,
+    CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS: 60000,
+    EXIT_CODE_RATE_LIMIT: 10,
+    EXIT_ON_RATE_LIMIT: true
   },
 
   DATA_FORMATTER: {
@@ -50,17 +53,6 @@ export const DEFAULT_CONFIG = {
         'mobile': 'mobile.json',
         '*': 'general.json'
       },
-    },
-    HARD_CODED_LINKS: [
-      // {
-      //   file_name: 'hc-links.json',
-      //   data: [
-      //     {
-      //       "url": "https://custom-link.com",
-      //       "content": "That's a custom link content, you can add as many as you want."
-      //     },
-      //   ]
-      // }
-    ]
+    }
   }
 };

package/src/loadConfig.js

@@ -16,10 +16,10 @@ export function loadConfig(userConfig = {}) {
   const config = deepMerge(DEFAULT_CONFIG, userConfig);
 
   // Dynamically construct paths using MAIN_DIR
-  config.CRAWLER.QUEUE_PATH = path.join(config.MAIN_DIR, 'queue.json');
-  config.CRAWLER.CRAWLED_PATH = path.join(config.MAIN_DIR, 'crawled');
-  config.DATA_FORMATTER.FORMATTED_PATH = path.join(config.MAIN_DIR, 'formatted');
-  config.DATA_FORMATTER.ERROR_REPORT_PATH = path.join(config.MAIN_DIR, 'error-report.json');
+  config.CRAWLER.QUEUE_PATH = path.posix.join(config.MAIN_DIR, 'queue.json');
+  config.CRAWLER.CRAWLED_PATH = path.posix.join(config.MAIN_DIR, 'crawled');
+  config.DATA_FORMATTER.FORMATTED_PATH = path.posix.join(config.MAIN_DIR, 'formatted');
+  config.DATA_FORMATTER.ERROR_REPORT_PATH = path.posix.join(config.MAIN_DIR, 'error-report.json');
 
   // If INCLUDE_URLS is not specified, set it to INITIAL_URLS by default
   if (!config.CRAWLER.INCLUDE_URLS || config.CRAWLER.INCLUDE_URLS.length === 0) {

package/src/scraply.js

@@ -5,7 +5,6 @@ import { processURL } from './utils/crawl/url/processor.js';
 import { formatData, saveSortedFormattedJSON } from './utils/format/formatData.js';
 
 let urlData = [];
-let urlMetadata = {};
 let CONFIG = {};
 let generatedFiles = new Set(); // Track files generated in the current crawl session.
 
@@ -17,7 +16,7 @@ const init = () => {
   
     CONFIG.CRAWLER.INITIAL_URLS.forEach(url => {
       const normalizedURL = normalizeURL(url);
-      urlData.push({ url: normalizedURL, file: null, status: null, error: null });
+      urlData.push({ url: normalizedURL, file: null, status: null, error: null, referrerUrl: null, depth: 0 });
     });
     saveQueue(urlData);
   } else { // If the queue is not empty
@@ -27,12 +26,10 @@ const init = () => {
 
       // Reset data for a fresh crawl.
       urlData = [];
-      urlMetadata = {};
 
       // Delete everything except CONFIG.DATA_FORMATTER.FORMATTED_PATH, so that the formatted data is always preserved until the crawler really finalizes the data.
       deleteDataFiles(CONFIG.CRAWLER.QUEUE_PATH);
       deleteDataFiles(CONFIG.CRAWLER.CRAWLED_PATH);
-      deleteDataFiles(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH);
 
       init();
     } else { // If there are URLs that haven't been processed yet, resume from the queue.
@@ -58,7 +55,7 @@ const start = async () => {
   let fileNumber = urlData.filter(entry => entry.file).length + 1;
   for await (const entry of urlData) {
     if (!entry.file) {
-      const processedFile = await processURL(entry, fileNumber, urlData, urlMetadata);
+      const processedFile = await processURL(entry, fileNumber, urlData);
       if (processedFile) {
         generatedFiles.add(processedFile); // Track the file generated
       }
@@ -71,7 +68,7 @@ const start = async () => {
   const notCrawledUrls = totalUrls - crawledUrls;
   const errorUrls = urlData.filter(entry => entry.error !== null);
 
-  console.log(`\nCRAWLING COMPLETED! ${crawledUrls} of ${totalUrls} (${notCrawledUrls} not crawled)`);
+  console.log(`\nCRAWLING COMPLETED! ${crawledUrls} of ${totalUrls} (${notCrawledUrls} not crawled, ${errorUrls.length} errors)`);
 
   // Iterate over all the urlData and save all the url & content to files, categorized by CONFIG.DATA_FORMATTER.CATEGORISED_PATHS. Exclude the URLs that match the patterns in CONFIG.DATA_FORMATTER.EXCLUDED_PATTERNS. Save in CONFIG.DATA_FORMATTER.FORMATTED_PATH.
   console.log(`\nFORMATTING DATA...`);
@@ -103,14 +100,6 @@ const start = async () => {
   };
   console.log(`${totalSavedURLs} total saved URLs to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
-  // Error reporting: Save into CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH the URLs that had any error: Save the url, the referrer, status code and error!
-  const errorData = errorUrls.map(entry => {
-    return { url: entry.url, status: entry.status, error: entry.error };
-  });
-  saveSortedFormattedJSON(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH, errorData);
-
-  console.log(`Errors: ${errorData.length} -> ${CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH}.`);
-
   // After formatting data, delete untracked files
   console.log(`\nCLEANING UP UNTRACKED FILES...`);
   deleteUntrackedFiles(CONFIG.DATA_FORMATTER.FORMATTED_PATH, generatedFiles); // Delete files not generated during this crawl

package/src/utils/crawl/fileOperations.js

@@ -11,7 +11,7 @@ export const saveJSON = (filePath, data) => {
 
 export const saveDataset = (data, fileNumber) => {
   if (!fs.existsSync(CONFIG.CRAWLER.CRAWLED_PATH)) fs.mkdirSync(CONFIG.CRAWLER.CRAWLED_PATH, { recursive: true });
-  const filename = `${CONFIG.CRAWLER.CRAWLED_PATH}/${fileNumber}.json`;
+  const filename = path.posix.join(CONFIG.CRAWLER.CRAWLED_PATH, `${fileNumber}.json`);
   fs.writeFileSync(filename, JSON.stringify(data, null, 2), 'utf8');
   return filename;
 };
@@ -22,7 +22,7 @@ export const deleteDataFiles = (filePath) => {
   if (fs.existsSync(filePath)) {
     if (fs.lstatSync(filePath).isDirectory()) {
       fs.readdirSync(filePath).forEach((file) => {
-        const currentPath = path.join(filePath, file);
+        const currentPath = path.posix.join(filePath, file);
         if (fs.lstatSync(currentPath).isDirectory()) {
           deleteDataFiles(currentPath);
         } else {
@@ -39,7 +39,7 @@ export const deleteDataFiles = (filePath) => {
 export const deleteUntrackedFiles = (folderPath, trackedFiles) => {
   if (fs.existsSync(folderPath)) {
     fs.readdirSync(folderPath).forEach((file) => {
-      const currentPath = path.join(folderPath, file);
+      const currentPath = path.posix.join(folderPath, file);
       if (fs.lstatSync(currentPath).isDirectory()) {
         deleteUntrackedFiles(currentPath, trackedFiles);
       } else if (!trackedFiles.has(currentPath)) {

package/src/utils/crawl/url/fetch.js

@@ -2,24 +2,41 @@ import axios from 'axios';
 import { delay } from '../delay.js';
 import { shouldRetry } from './handlers.js';
 
-export const fetchURL = async (url, retries = 2) => {
+export async function fetchURL(url, retries = 2) {
   try {
-    const response = await axios.get(url, { timeout: CONFIG.CRAWLER.REQUEST_TIMEOUT, maxRedirects: CONFIG.CRAWLER.MAX_REDIRECTS });
-    const contentType = response.headers['content-type'];
+    const response = await axios.get(url, {
+      timeout: CONFIG.CRAWLER.REQUEST_TIMEOUT,
+      maxRedirects: CONFIG.CRAWLER.MAX_REDIRECTS
+    });
 
+    const { 'content-type': contentType } = response.headers;
+
+    // Validate content type
     if (!CONFIG.CRAWLER.ALLOWED_CONTENT_TYPES.some(type => contentType.includes(type))) {
-      return { error: `Content-Type ${contentType} is not allowed.`, status: response.status };
+      return {
+        error: `Content-Type ${contentType} is not allowed.`,
+        status: response.status
+      };
     };
 
-    return { data: response.data, status: response.status };
+    return {
+      data: response.data,
+      status: response.status
+    };
   } catch (error) {
     if (retries > 0 && shouldRetry(error)) {
-      console.log(`Retrying (${CONFIG.CRAWLER.MAX_RETRIES - retries + 1}/${CONFIG.CRAWLER.MAX_RETRIES}) -> ${url}`);
+      const retryCount = CONFIG.CRAWLER.MAX_RETRIES - retries + 1;
+      console.log(`Retrying (${retryCount}/${CONFIG.CRAWLER.MAX_RETRIES}) -> ${url}`);
+      
       if (CONFIG.CRAWLER.CRAWL_ERROR_RETRY_DELAY_MS > 0) await delay(CONFIG.CRAWLER.CRAWL_ERROR_RETRY_DELAY_MS);
+      
       return fetchURL(url, retries - 1);
-    } else {
-      console.error(`Failed to fetch ${url} -> ${error.message}`);
-      return { error: error.message, status: error.response ? error.response.status : null };
+    }
+    
+    console.error(`Failed to fetch ${url} -> ${error.message}`);
+    return {
+      error: error.message,
+      status: error.response?.status
     };
   };
 };

package/src/utils/crawl/url/handlers.js

@@ -1,19 +1,40 @@
 import { URL } from 'node:url';
 import { normalizeURL } from './normalize.js';
+import { delay } from '../delay.js';
 
 // Handle HTML Status Codes HERE!
-export const shouldRetry = (error) => {
+export const shouldRetry = async (error) => {
   if (!error.response) return true;
-  if (error.response.status === 429) {
-    const waitTime = error.response.headers ? error.response.headers['retry-after'] : null;
-    if (waitTime) {
-      console.log(`Rate limited for ${waitTime} seconds, exiting Crawler...`);
+
+  const { status, headers } = error.response;
+  const retryAfter = headers?.['retry-after'];
+  const rateLimitReset = headers?.['x-ratelimit-reset'];
+
+  if (status === 429) {
+    let waitTime = null;
+
+    if (retryAfter) {
+      waitTime = isNaN(retryAfter)
+        ? Math.ceil((new Date(retryAfter).getTime() - Date.now()) / 1000) // HTTP date
+        : parseInt(retryAfter, 10); // Seconds
+      console.log(`Rate limited. Retrying after ${waitTime} seconds...`);
+    } else if (rateLimitReset) {
+      waitTime = Math.max(parseInt(rateLimitReset, 10) - Math.floow(Date.now() / 1000), 0);
+      console.log(`Rate limited. Retrying after ${waitTime} seconds...`);
     } else {
-      console.log(`Rate limited, no retry-after header found, exiting Crawler...`);
+      waitTime = CONFIG.CRAWLER.CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS / 1000;
+      console.log(`Rate limited. No 'retry-after' or 'x-ratelimit-reset' headers found. Falling back to ${waitTime} seconds...`);
+    }
+
+    if (CONFIG.CRAWLER.EXIT_ON_RATE_LIMIT) {
+      console.log(`Exiting due to rate limit.`);
+      process.exit(CONFIG.CRAWLER.EXIT_CODE_RATE_LIMIT); // GitHub Actions Docker uses values ranged from 0 to 255, so any bigger value will be % 256!
+    } else {
+      await delay(waitTime * 1000);
     }
-    process.exit(10); // GitHub Actions Docker uses values ranged from 0 to 255, so any bigger value will be % 256!
   }
-  return CONFIG.CRAWLER.RETRY_STATUS_CODES.includes(error.response.status); // Retry only on specific status codes
+
+  return CONFIG.CRAWLER.RETRY_STATUS_CODES.includes(status); // Retry on the specified status codes
 };
 
 const shouldIncludeURL = (url) => {
@@ -40,7 +61,7 @@ const shouldIncludeURL = (url) => {
   }
 };
 
-export const enqueueURLs = (urlData, urlMetadata, $, baseURL, referrer, depth) => {
+export const enqueueURLs = (urlData, $, baseURL, depth) => {
   $('a[href]').each((_, element) => {
     const href = $(element).attr('href');
     if (!href) return;
@@ -49,8 +70,7 @@ export const enqueueURLs = (urlData, urlMetadata, $, baseURL, referrer, depth) =
       const newURL = new URL(href, baseURL).toString();
       const normalizedURL = normalizeURL(newURL);
       if (shouldIncludeURL(normalizedURL) && !urlData.some(entry => entry.url === normalizedURL)) {
-        urlData.push({ url: normalizedURL, file: null, status: null, error: null });
-        urlMetadata[normalizedURL] = { referrer, depth };
+        urlData.push({ url: normalizedURL, file: null, status: null, error: null, referrerUrl: baseURL, depth });
       }
     } catch (error) {
       console.error(`Failed to enqueue URL: ${href} from ${baseURL}: ${error.message}`);

package/src/utils/crawl/url/processor.js

@@ -5,21 +5,20 @@ import { shouldRetry, enqueueURLs } from './handlers.js';
 import { fetchURL } from './fetch.js';
 import { saveDataset, saveQueue } from '../fileOperations.js';
 
-export const processURL = async (entry, fileNumber, urlData, urlMetadata) => {
-  if (entry.file || (entry.error && !shouldRetry({ response: { status: entry.status } }))) return;
+export const processURL = async (entry, fileNumber, urlData) => {
+  const startTime = new Date().getTime();
+  const { url, referrer, depth } = entry;
 
+  if (entry.file || (entry.error && !(await shouldRetry({ response: { status: entry.status } })))) return;
+  
   console.log(`- ${fileNumber}/${urlData.length} -> ${entry.url}`);
 
-  const { url } = entry;
-  const { referrer, depth } = urlMetadata[url] || { referrer: null, depth: 0 }; // Default depth is 0.
-
-  const startTime = new Date().getTime();
   try {
     const result = await fetchURL(url, CONFIG.CRAWLER.MAX_RETRIES);
     if (result && result.data) {
       const { data: html, status } = result;
       const $ = cheerio.load(html);
-      enqueueURLs(urlData, urlMetadata, $, url, url, depth + 1);
+      enqueueURLs(urlData, $, url, depth + 1);
 
       const content = cleanHTML($);
       const filename = saveDataset({ url, referrerURL: referrer, statusCode: status, depth, content }, fileNumber);

package/src/utils/format/formatData.js

@@ -24,7 +24,7 @@ export const formatData = (entry) => {
 
           // Fallback to wildcard match ('*') if no specific path is found
           if (!categorisedPath) categorisedPath = categorisedPaths['*'];
-          if (categorisedPath) return path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
+          if (categorisedPath) return path.posix.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
         }
       }
     } catch (e) {