scraply 1.0.10 → 1.0.11

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.10",
+  "version": "1.0.11",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/src/scraply.js

@@ -19,6 +19,9 @@ const init = () => {
       urlData.push({ url: normalizedURL, file: null, status: null, error: null });
     });
     saveQueue(urlData);
+
+    // Delete everything under CONFIG.MAIN_DIR
+    deleteDataFiles(CONFIG.MAIN_DIR);
   } else { // If the queue is not empty
     const allProcessed = urlData.every(entry => entry.file !== null || entry.error !== null);
     if (allProcessed) { // If all URLs have been processed
@@ -27,11 +30,12 @@ const init = () => {
       urlData = [];
       urlMetadata = {};
 
-      // Delete everything except CONFIG.DATA_FORMATTER.FORMATTED_PATH, so that the formatted data is always preserved until the crawler really finalizes the data. This way, the Discord Bot will fetch the correct & latest data from the GitHub repo, without fetching any incomplete data or empty data, as it watches for file diffs!
+      // Delete everything except CONFIG.DATA_FORMATTER.FORMATTED_PATH, so that the formatted data is always preserved until the crawler really finalizes the data.
       deleteDataFiles(CONFIG.CRAWLER.QUEUE_PATH);
       deleteDataFiles(CONFIG.CRAWLER.CRAWLED_PATH);
       deleteDataFiles(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH);
 
+
       init();
     } else { // If there are URLs that haven't been processed yet, resume from the queue.
       console.log(`Resuming from ${CONFIG.CRAWLER.QUEUE_PATH} with ${urlData.length} total found URLs\n`);