scraply 1.0.10 → 1.0.12

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.10",
+  "version": "1.0.12",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/src/scraply.js

@@ -1,12 +1,14 @@
 import { loadConfig } from './loadConfig.js';
 import { normalizeURL } from './utils/crawl/url/normalize.js';
-import { loadJSON, saveQueue, deleteDataFiles } from './utils/crawl/fileOperations.js';
+import { loadJSON, saveQueue, deleteDataFiles, deleteUntrackedFiles } from './utils/crawl/fileOperations.js';
 import { processURL } from './utils/crawl/url/processor.js';
 import { formatData, saveSortedFormattedJSON, saveHardcodedExtraLinks } from './utils/format/formatData.js';
+import path from 'node:path';
 
 let urlData = [];
 let urlMetadata = {};
 let CONFIG = {};
+let generatedFiles = new Set(); // Track files generated in the current crawl session.
 
 const init = () => {
   urlData = loadJSON(CONFIG.CRAWLER.QUEUE_PATH);
@@ -24,10 +26,11 @@ const init = () => {
     if (allProcessed) { // If all URLs have been processed
       console.log(`All URLs in ${CONFIG.CRAWLER.QUEUE_PATH} have been processed. Deleting persistent storage and starting a fresh Crawl...\n`);
 
+      // Reset data for a fresh crawl.
       urlData = [];
       urlMetadata = {};
 
-      // Delete everything except CONFIG.DATA_FORMATTER.FORMATTED_PATH, so that the formatted data is always preserved until the crawler really finalizes the data. This way, the Discord Bot will fetch the correct & latest data from the GitHub repo, without fetching any incomplete data or empty data, as it watches for file diffs!
+      // Delete everything except CONFIG.DATA_FORMATTER.FORMATTED_PATH, so that the formatted data is always preserved until the crawler really finalizes the data.
       deleteDataFiles(CONFIG.CRAWLER.QUEUE_PATH);
       deleteDataFiles(CONFIG.CRAWLER.CRAWLED_PATH);
       deleteDataFiles(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH);
@@ -55,9 +58,12 @@ const start = async () => {
 
   let fileNumber = urlData.filter(entry => entry.file).length + 1;
   for await (const entry of urlData) {
-    if (!entry.file) { // Only process URLs that haven't been processed yet.
-      await processURL(entry, fileNumber, urlData, urlMetadata);
-      fileNumber++; // Increment the file number only if the URL was processed successfully.
+    if (!entry.file) {
+      const processedFile = await processURL(entry, fileNumber, urlData, urlMetadata);
+      if (processedFile) {
+        generatedFiles.add(processedFile); // Track the file generated
+      }
+      fileNumber++;
     }
   }
 
@@ -94,6 +100,7 @@ const start = async () => {
     totalSavedURLs += data.length;
     console.log(`${data.length} -> ${savePath}`);
     saveSortedFormattedJSON(savePath, data);
+    generatedFiles.add(savePath); // Track the file saved
   };
   console.log(`${totalSavedURLs} total saved URLs to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
@@ -101,14 +108,24 @@ const start = async () => {
   const totalHardcodedLinks = await saveHardcodedExtraLinks();
   console.log(`${totalHardcodedLinks} Hardcoded extra links saved to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
+  // Track the files generated for hardcoded links with full paths.
+  CONFIG.DATA_FORMATTER.HARD_CODED_LINKS.forEach(link => {
+    const hardcodedFilePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, link.file_name);
+    generatedFiles.add(hardcodedFilePath); // Add the full path to the set
+  });
+
   // Error reporting: Save into CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH the URLs that had any error: Save the url, the referrer, status code and error!
   const errorData = errorUrls.map(entry => {
     return { url: entry.url, status: entry.status, error: entry.error };
   });
-
   saveSortedFormattedJSON(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH, errorData);
 
   console.log(`Errors: ${errorData.length} -> ${CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH}.`);
+
+  // After formatting data, delete untracked files
+  console.log(`\nCLEANING UP UNTRACKED FILES...`);
+  deleteUntrackedFiles(CONFIG.DATA_FORMATTER.FORMATTED_PATH, generatedFiles); // Delete files not generated during this crawl
+  generatedFiles.clear(); // Clear the set to prepare for future crawls
 };
 
 // Main function to be exported and used

package/src/utils/crawl/fileOperations.js

@@ -35,3 +35,17 @@ export const deleteDataFiles = (filePath) => {
     }
   }
 };
+
+export const deleteUntrackedFiles = (folderPath, trackedFiles) => {
+  if (fs.existsSync(folderPath)) {
+    fs.readdirSync(folderPath).forEach((file) => {
+      const currentPath = path.join(folderPath, file);
+      if (fs.lstatSync(currentPath).isDirectory()) {
+        deleteUntrackedFiles(currentPath, trackedFiles);
+      } else if (!trackedFiles.has(currentPath)) {
+        console.log(`Deleting untracked file: ${currentPath}`);
+        fs.unlinkSync(currentPath);
+      }
+    });
+  }
+};

package/src/utils/crawl/url/processor.js

@@ -1,6 +1,5 @@
 import { delay } from '../delay.js';
 import { cleanHTML } from '../cleanHTML.js';
-
 import * as cheerio from 'cheerio';
 import { shouldRetry, enqueueURLs } from './handlers.js';
 import { fetchURL } from './fetch.js';
@@ -13,7 +12,7 @@ export const processURL = async (entry, fileNumber, urlData, urlMetadata) => {
 
   const { url } = entry;
   const { referrer, depth } = urlMetadata[url] || { referrer: null, depth: 0 }; // Default depth is 0.
-  
+
   const startTime = new Date().getTime();
   try {
     const result = await fetchURL(url, CONFIG.CRAWLER.MAX_RETRIES);
@@ -21,11 +20,15 @@ export const processURL = async (entry, fileNumber, urlData, urlMetadata) => {
       const { data: html, status } = result;
       const $ = cheerio.load(html);
       enqueueURLs(urlData, urlMetadata, $, url, url, depth + 1);
+
       const content = cleanHTML($);
       const filename = saveDataset({ url, referrerURL: referrer, statusCode: status, depth, content }, fileNumber);
+
       entry.file = filename;
       entry.status = status;
       entry.error = null;
+
+      return filename; // Return the filename of the saved dataset
     } else {
       entry.error = result.error;
       entry.status = result.status;
@@ -33,12 +36,18 @@ export const processURL = async (entry, fileNumber, urlData, urlMetadata) => {
   } catch (error) {
     entry.error = error.message;
     entry.status = null;
-  }
+  } finally {
+    // Save the queue state whether successful or not
+    saveQueue(urlData);
 
-  saveQueue(urlData);
+    const endTime = new Date().getTime();
+    const elapsedTime = endTime - startTime;
 
-  const endTime = new Date().getTime();
-  const elapsedTime = endTime - startTime;
+    // Apply delay if necessary
+    if (CONFIG.CRAWLER.CRAWL_DELAY_MS > 0 && elapsedTime < CONFIG.CRAWLER.CRAWL_DELAY_MS) {
+      await delay(CONFIG.CRAWLER.CRAWL_DELAY_MS - elapsedTime);
+    }
+  }
 
-  if (CONFIG.CRAWLER.CRAWL_DELAY_MS > 0 && elapsedTime < CONFIG.CRAWLER.CRAWL_DELAY_MS) await delay(CONFIG.CRAWLER.CRAWL_DELAY_MS - elapsedTime);
+  return null; // Return null if no file was generated
 };