scraply 1.0.1 → 1.0.3

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/readme.md

@@ -1,4 +1,89 @@
+# Scraply
 Scraply is a customizable and efficient web crawler and data scraper for Node.js, designed to handle various web crawling needs with ease. You can define the URLs to crawl, configure patterns to include/exclude, and format the output data in JSON. Scraply is built to be flexible, with user-configurable settings and dynamic paths.
 
-Installation
-`npm install scraply`
+Bug Reports & Dev Stuff on: [Scraply's GitHub](https://github.com/pauserratgutierrez/scraply)
+
+## Installation
+Using npm:
+``npm install scraply``
+
+## Working Example
+Initialize Scraply with provided URLs to start crawling:
+```
+import { scraply } from 'scraply';
+scraply({
+  CRAWLER: {
+    INITIAL_URLS: ['https://example.com']
+  }
+});
+```
+
+## How Scraply Works
+### Persistent Data Storage
+Scraply persistently saves the state of the crawler in JSON files (the queue, crawled data, etc.). If the crawler is interrupted or rate-limited, all progress is saved, and the crawler will automatically stop. When restarted, Scraply resumes crawling exactly where it left off, without reprocessing already crawled URLs.
+
+### Handling Rate Limiting
+Scraply is designed to handle rate-limiting gracefully. If the crawler encounters rate-limited responses (e.g., status code `429`), it stops processing further requests and saves everything in the queue. Once restarted, it resumes the crawling process from where it stopped.
+
+This makes Scraply ideal for long-running, continuous crawling tasks. You can integrate Scraply with GitHub Actions or other CI/CD pipelines to perform endless crawling jobs over time. Simply schedule Scraply to run periodically, and it will continue gathering data without duplicating work.
+
+### Integration with GitHub Actions
+Scraply can be easily integrated into a GitHub Action workflow for continuous, long-running crawling tasks. You can set it up to crawl for a set duration or number of URLs, persistently saving the progress, and then resuming where it left off on the next run.
+
+## Config Options
+Scraply allows you to pass a configuration object to the main ```scraply()``` function to customize the crawling behavior. Below are the current configuration options:
+```
+MAIN_DIR: 'dataset',
+
+CRAWLER: {
+  INITIAL_URLS: [
+    'https://crawler-test.com/'
+  ],
+  INCLUDE_URLS: [
+    'https://crawler-test.com/.*'
+  ],
+  ALLOWED_CONTENT_TYPES: [
+    'text/html'
+  ],
+  EXCLUDE_PATTERNS: [
+    '/cdn-cgi/',
+    /\.(zip|rar|webp|png|jpg|jpeg|gif|mp3|mp4|pdf|css|js|svg|ico|eot|ttf|woff|woff2|otf|webm|ogg|wav|flac|m4a|mkv|mov|avi|wmv|flv|swf|exe|msi|dmg|iso|bin)$/,
+  ],
+  DOM_ELEMENTS_REMOVE: [
+    'script',
+    'noscript',
+    'style',
+    'meta',
+    'link',
+    'svg',
+    'path',
+    'img',
+    'input',
+    'textarea',
+    'embed',
+    'object',
+    'iframe',
+    'nav',
+    'header',
+    'footer',
+    'aside',
+    'button'
+  ],
+  RETRY_STATUS_CODES: [408, 429, 500, 502, 503, 504],
+  REQUEST_TIMEOUT: 4000,
+  MAX_REDIRECTS: 3,
+  MAX_RETRIES: 2,
+  CRAWL_DELAY_MS: 200,
+  CRAWL_ERROR_RETRY_DELAY_MS: 800,
+},
+
+DATA_FORMATTER: {
+  EXCLUDED_PATTERNS: [],
+  CATEGORISED_PATHS: {
+    'https://crawler-test.com': {
+      'mobile': 'mobile.json',
+      'fallback': 'general.json'
+    },
+  },
+}
+```

package/src/loadConfig.js

@@ -1,5 +1,5 @@
 import path from 'node:path';
-import { DEFAULT_CONFIG } from './config.js';
+import { DEFAULT_CONFIG } from './defaultConfig.js';
 
 // A utility function to perform a deep merge of objects
 function deepMerge(target, source) {
@@ -21,5 +21,10 @@ export function loadConfig(userConfig = {}) {
   config.DATA_FORMATTER.FORMATTED_PATH = path.join(config.MAIN_DIR, 'formatted');
   config.DATA_FORMATTER.ERROR_REPORT_PATH = path.join(config.MAIN_DIR, 'error-report.json');
 
+  // If INCLUDE_URLS is not specified, set it to INITIAL_URLS by default
+  if (!config.CRAWLER.INCLUDE_URLS || config.CRAWLER.INCLUDE_URLS.length === 0) {
+    config.CRAWLER.INCLUDE_URLS = config.CRAWLER.INITIAL_URLS.map(url => `${url}.*`);
+  }
+
   return config;
 };

package/src/scraply.js

@@ -4,16 +4,11 @@ import { loadJSON, saveQueue, deleteDataFiles } from './utils/crawl/fileOperatio
 import { processURL } from './utils/crawl/url/processor.js';
 import { formatData, saveSortedFormattedJSON, saveHardcodedExtraLinks } from './utils/format/formatData.js';
 
-const userConfig = {};
-
-// Load and merge the configuration
-const CONFIG = loadConfig(userConfig);
-global.CONFIG = CONFIG;
-
 let urlData = [];
 let urlMetadata = {};
+let CONFIG = {};
 
-export const initializeCrawler = () => {
+const init = () => {
   urlData = loadJSON(CONFIG.CRAWLER.QUEUE_PATH);
 
   if (urlData.length === 0) { // If the queue is empty, start fresh with the initial URLs.
@@ -37,14 +32,14 @@ export const initializeCrawler = () => {
       deleteDataFiles(CONFIG.CRAWLER.CRAWLED_PATH);
       deleteDataFiles(CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH);
 
-      initializeCrawler();
+      init();
     } else { // If there are URLs that haven't been processed yet, resume from the queue.
       console.log(`Resuming from ${CONFIG.CRAWLER.QUEUE_PATH} with ${urlData.length} total found URLs\n`);
     }
   }
 };
 
-export const scraply = async () => {
+const start = async () => {
   console.log(`STARTING CRAWLER
   - Initial URLs: ${CONFIG.CRAWLER.INITIAL_URLS}
   - Include URLs: ${CONFIG.CRAWLER.INCLUDE_URLS}
@@ -115,3 +110,12 @@ export const scraply = async () => {
 
   console.log(`Errors: ${errorData.length} -> ${CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH}.`);
 };
+
+// Main function to be exported and used
+export const scraply = async (userConfig = {}) => {
+  CONFIG = loadConfig(userConfig);
+  global.CONFIG = CONFIG;
+
+  init();
+  await start();
+};