scraply 1.0.11 → 1.0.13

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.11",
+  "version": "1.0.13",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/src/scraply.js

@@ -1,12 +1,14 @@
 import { loadConfig } from './loadConfig.js';
 import { normalizeURL } from './utils/crawl/url/normalize.js';
-import { loadJSON, saveQueue, deleteDataFiles } from './utils/crawl/fileOperations.js';
+import { loadJSON, saveQueue, deleteDataFiles, deleteUntrackedFiles } from './utils/crawl/fileOperations.js';
 import { processURL } from './utils/crawl/url/processor.js';
 import { formatData, saveSortedFormattedJSON, saveHardcodedExtraLinks } from './utils/format/formatData.js';
+import path from 'node:path';
 
 let urlData = [];
 let urlMetadata = {};
 let CONFIG = {};
+let generatedFiles = new Set(); // Track files generated in the current crawl session.
 
 const init = () => {
   urlData = loadJSON(CONFIG.CRAWLER.QUEUE_PATH);
@@ -19,14 +21,12 @@ const init = () => {
       urlData.push({ url: normalizedURL, file: null, status: null, error: null });
     });
     saveQueue(urlData);
-
-    // Delete everything under CONFIG.MAIN_DIR
-    deleteDataFiles(CONFIG.MAIN_DIR);
   } else { // If the queue is not empty
     const allProcessed = urlData.every(entry => entry.file !== null || entry.error !== null);
     if (allProcessed) { // If all URLs have been processed
       console.log(`All URLs in ${CONFIG.CRAWLER.QUEUE_PATH} have been processed. Deleting persistent storage and starting a fresh Crawl...\n`);
 
+      // Reset data for a fresh crawl.
       urlData = [];
       urlMetadata = {};
 
@@ -35,7 +35,6 @@ const init = () => {
       deleteDataFiles(CONFIG.CRAWLER.CRAWLED_PATH);
       deleteDataFiles(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH);
 
-
       init();
     } else { // If there are URLs that haven't been processed yet, resume from the queue.
       console.log(`Resuming from ${CONFIG.CRAWLER.QUEUE_PATH} with ${urlData.length} total found URLs\n`);
@@ -59,9 +58,12 @@ const start = async () => {
 
   let fileNumber = urlData.filter(entry => entry.file).length + 1;
   for await (const entry of urlData) {
-    if (!entry.file) { // Only process URLs that haven't been processed yet.
-      await processURL(entry, fileNumber, urlData, urlMetadata);
-      fileNumber++; // Increment the file number only if the URL was processed successfully.
+    if (!entry.file) {
+      const processedFile = await processURL(entry, fileNumber, urlData, urlMetadata);
+      if (processedFile) {
+        generatedFiles.add(processedFile); // Track the file generated
+      }
+      fileNumber++;
     }
   }
 
@@ -98,6 +100,7 @@ const start = async () => {
     totalSavedURLs += data.length;
     console.log(`${data.length} -> ${savePath}`);
     saveSortedFormattedJSON(savePath, data);
+    generatedFiles.add(savePath); // Track the file saved
   };
   console.log(`${totalSavedURLs} total saved URLs to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
@@ -105,14 +108,24 @@ const start = async () => {
   const totalHardcodedLinks = await saveHardcodedExtraLinks();
   console.log(`${totalHardcodedLinks} Hardcoded extra links saved to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
+  // Track the files generated for hardcoded links with full paths.
+  CONFIG.DATA_FORMATTER.HARD_CODED_LINKS.forEach(link => {
+    const hardcodedFilePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, link.file_name);
+    generatedFiles.add(hardcodedFilePath); // Add the full path to the set
+  });
+
   // Error reporting: Save into CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH the URLs that had any error: Save the url, the referrer, status code and error!
   const errorData = errorUrls.map(entry => {
     return { url: entry.url, status: entry.status, error: entry.error };
   });
-
   saveSortedFormattedJSON(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH, errorData);
 
   console.log(`Errors: ${errorData.length} -> ${CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH}.`);
+
+  // After formatting data, delete untracked files
+  console.log(`\nCLEANING UP UNTRACKED FILES...`);
+  deleteUntrackedFiles(CONFIG.DATA_FORMATTER.FORMATTED_PATH, generatedFiles); // Delete files not generated during this crawl
+  generatedFiles.clear(); // Clear the set to prepare for future crawls
 };
 
 // Main function to be exported and used

package/src/utils/crawl/fileOperations.js

@@ -35,3 +35,17 @@ export const deleteDataFiles = (filePath) => {
     }
   }
 };
+
+export const deleteUntrackedFiles = (folderPath, trackedFiles) => {
+  if (fs.existsSync(folderPath)) {
+    fs.readdirSync(folderPath).forEach((file) => {
+      const currentPath = path.join(folderPath, file);
+      if (fs.lstatSync(currentPath).isDirectory()) {
+        deleteUntrackedFiles(currentPath, trackedFiles);
+      } else if (!trackedFiles.has(currentPath)) {
+        console.log(`Deleting untracked file: ${currentPath}`);
+        fs.unlinkSync(currentPath);
+      }
+    });
+  }
+};

package/src/utils/crawl/url/processor.js

@@ -1,6 +1,5 @@
 import { delay } from '../delay.js';
 import { cleanHTML } from '../cleanHTML.js';
-
 import * as cheerio from 'cheerio';
 import { shouldRetry, enqueueURLs } from './handlers.js';
 import { fetchURL } from './fetch.js';
@@ -13,7 +12,7 @@ export const processURL = async (entry, fileNumber, urlData, urlMetadata) => {
 
   const { url } = entry;
   const { referrer, depth } = urlMetadata[url] || { referrer: null, depth: 0 }; // Default depth is 0.
-  
+
   const startTime = new Date().getTime();
   try {
     const result = await fetchURL(url, CONFIG.CRAWLER.MAX_RETRIES);
@@ -21,11 +20,15 @@ export const processURL = async (entry, fileNumber, urlData, urlMetadata) => {
       const { data: html, status } = result;
       const $ = cheerio.load(html);
       enqueueURLs(urlData, urlMetadata, $, url, url, depth + 1);
+
       const content = cleanHTML($);
       const filename = saveDataset({ url, referrerURL: referrer, statusCode: status, depth, content }, fileNumber);
+
       entry.file = filename;
       entry.status = status;
       entry.error = null;
+
+      return filename; // Return the filename of the saved dataset
     } else {
       entry.error = result.error;
       entry.status = result.status;
@@ -33,12 +36,18 @@ export const processURL = async (entry, fileNumber, urlData, urlMetadata) => {
   } catch (error) {
     entry.error = error.message;
     entry.status = null;
-  }
+  } finally {
+    // Save the queue state whether successful or not
+    saveQueue(urlData);
 
-  saveQueue(urlData);
+    const endTime = new Date().getTime();
+    const elapsedTime = endTime - startTime;
 
-  const endTime = new Date().getTime();
-  const elapsedTime = endTime - startTime;
+    // Apply delay if necessary
+    if (CONFIG.CRAWLER.CRAWL_DELAY_MS > 0 && elapsedTime < CONFIG.CRAWLER.CRAWL_DELAY_MS) {
+      await delay(CONFIG.CRAWLER.CRAWL_DELAY_MS - elapsedTime);
+    }
+  }
 
-  if (CONFIG.CRAWLER.CRAWL_DELAY_MS > 0 && elapsedTime < CONFIG.CRAWLER.CRAWL_DELAY_MS) await delay(CONFIG.CRAWLER.CRAWL_DELAY_MS - elapsedTime);
+  return null; // Return null if no file was generated
 };

package/src/utils/format/formatData.js

@@ -9,17 +9,22 @@ export const formatData = (entry) => {
       const isExcluded = CONFIG.DATA_FORMATTER.EXCLUDED_PATTERNS.some(pattern => new RegExp(pattern).test(entry.url));
   
       if (!isExcluded) {
-        // Check for the specific category path
-        const pathSegments = pathname.split('/');
-        let categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.[pathSegments[1]];
-        
-        // If no specific category path is found, use the "*" fallback
-        if (!categorisedPath) {
-          categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.['*'];
-        }
-
-        if (categorisedPath) {
-          return path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
+        const pathSegments = pathname.split('/').filter(Boolean); // filter out empty segments
+        const categorisedPaths = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin];
+
+        if (categorisedPaths) {
+          // Try to match the full path segments, reducing specificity step by step
+          let categorisedPath = null;
+
+          for (let i = pathSegments.length; i >= 1; i--) {
+            const pathKey = pathSegments.slice(0, i).join('/');
+            categorisedPath = categorisedPaths[pathKey];
+            if (categorisedPath) break;
+          }
+
+          // Fallback to wildcard match ('*') if no specific path is found
+          if (!categorisedPath) categorisedPath = categorisedPaths['*'];
+          if (categorisedPath) return path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
         }
       }
     } catch (e) {