scraply 1.0.0 → 1.0.2

package/package.json

@@ -1,17 +1,20 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {
     "start": "node ."
   },
-  "keywords": ["crawler", "scraper"],
+  "keywords": [
+    "crawler",
+    "scraper"
+  ],
   "author": "Pau Serrat Gutiérrez",
   "dependencies": {
     "axios": "^1.7.7",
     "cheerio": "^1.0.0",
     "he": "^1.2.0"
   }
-}
+}

package/readme.md

@@ -0,0 +1,4 @@
+Scraply is a customizable and efficient web crawler and data scraper for Node.js, designed to handle various web crawling needs with ease. You can define the URLs to crawl, configure patterns to include/exclude, and format the output data in JSON. Scraply is built to be flexible, with user-configurable settings and dynamic paths.
+
+Installation
+`npm install scraply`

package/src/scraply.js

@@ -4,14 +4,9 @@ import { loadJSON, saveQueue, deleteDataFiles } from './utils/crawl/fileOperatio
 import { processURL } from './utils/crawl/url/processor.js';
 import { formatData, saveSortedFormattedJSON, saveHardcodedExtraLinks } from './utils/format/formatData.js';
 
-const userConfig = {};
-
-// Load and merge the configuration
-const CONFIG = loadConfig(userConfig);
-global.CONFIG = CONFIG;
-
 let urlData = [];
 let urlMetadata = {};
+let CONFIG = {};
 
 const initializeCrawler = () => {
   urlData = loadJSON(CONFIG.CRAWLER.QUEUE_PATH);
@@ -44,7 +39,7 @@ const initializeCrawler = () => {
   }
 };
 
-const app = async () => {
+const crawler = async () => {
   console.log(`STARTING CRAWLER
   - Initial URLs: ${CONFIG.CRAWLER.INITIAL_URLS}
   - Include URLs: ${CONFIG.CRAWLER.INCLUDE_URLS}
@@ -116,5 +111,11 @@ const app = async () => {
   console.log(`Errors: ${errorData.length} -> ${CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH}.`);
 };
 
-initializeCrawler();
-await app();
+// Main function to be exported and used
+export const scraply = async (userConfig = {}) => {
+  CONFIG = loadConfig(userConfig);
+  global.CONFIG = CONFIG;
+
+  initializeCrawler();
+  await crawler();
+};