scraply 1.0.25 → 2.0.1

package/package.json

@@ -1,26 +1,43 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.25",
-  "main": "src/scraply.js",
+  "version": "2.0.1",
+  "main": "src/index.js",
   "type": "module",
+  "exports": {
+    ".": "./src/index.js"
+  },
+  "files": [
+    "src"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
   "scripts": {
-    "start": "node ."
+    "dev": "node src/dev.js"
   },
   "keywords": [
     "crawler",
-    "scraper"
+    "scraper",
+    "web-scraping",
+    "puppeteer",
+    "cheerio"
   ],
   "author": "Pau Serrat Gutiérrez",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/pauserratgutierrez/scraply.git"
+  },
   "dependencies": {
-    "axios": "^1.7.9",
-    "cheerio": "^1.0.0",
-    "he": "^1.2.0",
-    "puppeteer": "^24.2.0",
-    "puppeteer-cluster": "^0.24.0"
+    "cheerio": "1.2.0",
+    "puppeteer": "25.1.0",
+    "puppeteer-cluster": "0.25.0"
   },
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
+  },
+  "allowScripts": {
+    "puppeteer@25.1.0": true
   }
 }

package/readme.md

@@ -1,94 +1,162 @@
 # Scraply
-Scraply is a customizable and efficient web crawler and data scraper for Node.js, designed to handle various web crawling needs with ease. You can define the URLs to crawl, configure patterns to include/exclude, and format the output data in JSON. Scraply is built to be flexible, with user-configurable settings and dynamic paths.
 
-Bug Reports & Dev Stuff on: [Scraply's GitHub](https://github.com/pauserratgutierrez/scraply)
-NPM Package: [Scraply's NPM](https://www.npmjs.com/package/scraply)
+Scraply is a customizable, modular web crawler and content scraper for Node.js. Define the URLs to crawl, control which links are followed, choose how pages are fetched (plain HTTP or a real browser), and route the extracted text into JSON files. Crawls are persistent and resumable, so they are well suited to long-running or scheduled jobs.
 
-## Installation
-Using npm:
-``npm install scraply``
+Bug reports and development: [Scraply on GitHub](https://github.com/pauserratgutierrez/scraply)
+NPM package: [Scraply on NPM](https://www.npmjs.com/package/scraply)
+
+> Scraply 2.0 is a ground-up rewrite with a new configuration shape and public API. See [Migrating from 1.x](#migrating-from-1x).
+
+## Requirements
+- Node.js >= 18 (uses the built-in `fetch`).
 
-## Working Example
-Initialize Scraply with provided URLs to start crawling:
+## Installation
+```
+npm install scraply
 ```
+
+## Quick start
+```js
 import { scraply } from 'scraply';
-scraply({
-  CRAWLER: {
-    INITIAL_URLS: ['https://example.com']
+
+await scraply({
+  startUrls: ['https://example.com'],
+  output: {
+    routes: {
+      'https://example.com': { '*': 'example.json' }
+    }
   }
 });
 ```
 
-## How Scraply Works
-### Persistent Data Storage
-Scraply persistently saves the state of the crawler in JSON files (the queue, crawled data, etc.). If the crawler is interrupted or rate-limited, all progress is saved, and the crawler will automatically stop. When restarted, Scraply resumes crawling exactly where it left off, without reprocessing already crawled URLs.
+This crawls `example.com`, extracts the readable text of every allowed page, and writes the results to `dataset/formatted/example.json`.
+
+## How Scraply works
+1. The crawl is seeded from `startUrls`.
+2. Each page is fetched, its links are discovered and filtered (`include` / `exclude`), and new links are queued.
+3. The page text is extracted (configurable element removal) and saved under `dataset/crawled/` as `{ url, content, crawledAt, hash }` (`crawledAt` is an ISO timestamp; `hash` is the SHA-256 of `content`, handy for change detection).
+4. When the queue drains, all crawled pages are routed by URL into the files defined in `output.routes` and written to `dataset/formatted/`.
+
+Each queue entry ends in one of three terminal states: **crawled** (saved), **skipped** (disallowed `Content-Type`), or **error** (fetch failed). The three are tracked separately so stats stay meaningful.
+
+### Persistence and resuming
+The queue and crawled pages are checkpointed to disk in `dataset/`. If a run is interrupted (or rate-limited), progress is saved and the next run resumes exactly where it left off without re-crawling finished URLs. When every URL has been processed, Scraply starts a fresh crawl (set `crawl.resetOnComplete: false` to keep the finished queue instead). To re-attempt failed URLs on the next run, set `crawl.retryErrors: true` (or call `requeueErrors()` and crawl again).
+
+### Concurrency and limits
+Pages are crawled with a worker pool (`crawl.concurrency`). Requests to the same host are spaced by `crawl.delay` for politeness, while different hosts run in parallel. `crawl.maxDepth` bounds link depth and `crawl.maxPages` caps the total number of successfully crawled pages (counted across resumes).
+
+### Rate limiting
+On HTTP `429`, Scraply either exits immediately with `rateLimit.exitCode` (default) so a scheduler can retry later, or waits (honoring `retry-after` / `x-ratelimit-reset`) and retries — independently of the normal `retry` budget — when `rateLimit.exitOnLimit` is `false`.
+
+## Fetchers
+`fetcher` selects the backend:
+- `'http'` (default): fast static fetching with the native `fetch`. Redirects are followed up to `request.maxRedirects`, and response bodies larger than `request.maxContentLength` (default 20 MB, `0` disables) are rejected before they are buffered.
+- `'browser'`: full JavaScript rendering via Puppeteer (`puppeteer-cluster`).
+- a custom object implementing the `Fetcher` interface (`{ name, fetch, init?, close? }`), so backends like Playwright or a remote CDP browser can be plugged in without changing the crawler.
+
+Both built-in fetchers send `request.userAgent` and any extra `request.headers` (e.g. `Authorization`, `Accept-Language`, `Cookie`) with every request.
+
+### Browser fetcher options
+The `browser` block applies only when `fetcher: 'browser'`. Both options are validated at config load time. See [`src/config/defaults.js`](src/config/defaults.js) for defaults.
+
+- **`browser.waitUntil`** — passed to Puppeteer `page.goto`. Default `'load'`. Use `'networkidle2'` for SPAs that inject links or content after the initial load (Vue/React sites). Increase `request.timeout` when using slower modes.
+- **`browser.blockResources`** — Puppeteer resource types to abort during fetch (`'image'`, `'stylesheet'`, `'font'`, `'media'`). Default `['image', 'font', 'media']`. Stylesheets are excluded by default because many SPAs need CSS before content renders. Pass `[]` to disable resource blocking entirely.
+
+```js
+await scraply({
+  startUrls: ['https://spa.example.com/products'],
+  fetcher: 'browser',
+  browser: {
+    waitUntil: 'networkidle2',
+    blockResources: ['image', 'font', 'media']
+  },
+  request: { timeout: 60000 }
+});
+```
+
+## Programmatic API
+`createCrawler(config)` returns an instance exposing each stage, plus lifecycle hooks:
+
+```js
+import { createCrawler } from 'scraply';
+
+const crawler = createCrawler({ startUrls: ['https://example.com'] });
+
+// React to every crawled page as it happens.
+crawler.on('page', (record) => console.log('crawled', record.url));
+
+// Veto links before they are queued.
+crawler.on('shouldEnqueue', (url) => !url.includes('/admin'));
+
+// Transform the stored record.
+crawler.on('transform', (record) => ({ ...record, length: record.content.length }));
+
+await crawler.run();
+```
+
+Instance methods: `run()`, `crawl()`, `fetch(url)`, `extract(html, url)`, `enqueue(urls, opts)`, `format(records?)`, `requeueErrors()`, `stop()`, `on(event, fn)`.
+
+`format()` reads crawled pages from `dataset/crawled/` via the persisted queue. You can call it alone to re-route output after a crawl — no need to fetch pages again.
+
+Hooks: `response`, `extract`, `shouldEnqueue`, `transform`, `page`, `error`.
 
-### Handling Rate Limiting
-Scraply is designed to handle rate-limiting gracefully. If the crawler encounters rate-limited responses (e.g., status code `429`), it stops processing further requests and saves everything in the queue. Once restarted, it resumes the crawling process from where it stopped.
+Standalone exports for advanced use: `normalizeUrl`, `matchesPattern`, `matchesAnyPattern`, `extractText`, `discoverLinks`, `routeRecord`, `writeRecords`, `formatRecords`, `loadConfig`, `DEFAULT_CONFIG`, `resolveFetcher`, `createHttpFetcher`, `createBrowserFetcher`, `assertBrowserConfig`, `BROWSER_WAIT_UNTIL`, `BROWSER_BLOCKABLE_RESOURCES`.
 
-This makes Scraply ideal for long-running, continuous crawling tasks. You can integrate Scraply with GitHub Actions or other CI/CD pipelines to perform endless crawling jobs over time. Simply schedule Scraply to run periodically, and it will continue gathering data without duplicating work.
+## Configuration
+All options are optional except `startUrls`. Pass a partial object to `scraply()` or `createCrawler()` — it is [deep-merged](src/config/load.js) over the defaults. Durations are in milliseconds.
 
-### Integration with GitHub Actions
-Scraply can be easily integrated into a GitHub Action workflow for continuous, long-running crawling tasks. You can set it up to crawl for a set duration or number of URLs, persistently saving the progress, and then resuming where it left off on the next run.
+**Full default values and inline comments:** [`src/config/defaults.js`](src/config/defaults.js)
 
-## Config Options
-Scraply allows you to pass a configuration object to the main ```scraply()``` function to customize the crawling behavior. Below are the current configuration options:
+```js
+import { DEFAULT_CONFIG, loadConfig } from 'scraply';
+
+// Inspect or extend the defaults programmatically.
+const config = loadConfig({
+  ...DEFAULT_CONFIG,
+  startUrls: ['https://example.com']
+});
 ```
-MAIN_DIR: 'dataset',
-
-CRAWLER: {
-  INITIAL_URLS: [
-    'https://crawler-test.com/'
-  ],
-  INCLUDE_URLS: [
-    'https://crawler-test.com/'
-  ],
-  ALLOWED_CONTENT_TYPES: [
-    'text/html'
-  ],
-  EXCLUDE_PATTERNS: [
-    '/cdn-cgi/',
-    /\.(zip|rar|webp|png|jpg|jpeg|gif|mp3|mp4|pdf|css|js|svg|ico|eot|ttf|woff|woff2|otf|webm|ogg|wav|flac|m4a|mkv|mov|avi|wmv|flv|swf|exe|msi|dmg|iso|bin)$/,
-  ],
-  DOM_ELEMENTS_REMOVE: [
-    'script',
-    'noscript',
-    'style',
-    'meta',
-    'link',
-    'svg',
-    'path',
-    'img',
-    'input',
-    'textarea',
-    'embed',
-    'object',
-    'iframe',
-    'nav',
-    'header',
-    'footer',
-    'aside',
-    'button'
-  ],
-  RETRY_STATUS_CODES: [408, 500, 502, 503, 504],
-  REQUEST_TIMEOUT: 3000,
-  MAX_REDIRECTS: 2,
-  MAX_CONTENT_LENGTH: 20 * 1024 * 1024, // 20MB
-  MAX_RETRIES: 1,
-  CRAWL_DELAY_MS: 200,
-  CRAWL_ERROR_RETRY_DELAY_MS: 1000,
-  CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS: 60000,
-  EXIT_ON_RATE_LIMIT: true, // If true, forces exit instantly. If false, only exits after retries (if still 429)
-  EXIT_CODE_RATE_LIMIT: 10
-},
-
-DATA_FORMATTER: {
-  EXCLUDED_PATTERNS: [],
-  CATEGORISED_PATHS: {
-    'https://crawler-test.com': {
-      'mobile': 'mobile.json',
-      '*': 'general.json'
+
+Top-level keys: `startUrls`, `include`, `exclude`, `allowedContentTypes`, `fetcher`, `browser`, `logLevel`, `storage`, `request`, `retry`, `rateLimit`, `crawl`, `extract`, `output`.
+
+### Output routing
+`output.routes` is a two-level map:
+
+1. **Outer keys** — URL prefix (usually `https://origin`, or `https://origin/path`) matched against the full crawled URL.
+2. **Inner keys** — pathname segments joined with `/`, **without a leading slash**, matched from the longest suffix upward. Use `'*'` as fallback within that prefix.
+
+Inner keys are case-sensitive and must match the URL pathname exactly (e.g. `Products/sports-watches`, not `/products/sports-watches`).
+
+```js
+output: {
+  routes: {
+    'https://docs.example.com': {
+      'guide': 'guides.json',
+      '*': 'docs.json'
     },
+    'https://example.com/products/sports-watches': { '*': 'watches.json' }
   }
 }
-```
+```
+
+## GitHub Actions
+Because crawls are persistent and exit cleanly on rate limits, Scraply works well on a schedule. Commit the `dataset/` directory between runs, and each scheduled run continues the crawl.
+
+## Migrating from 1.x
+The configuration is now camelCase and grouped, and the entry point is `src/index.js`.
+
+- `MAIN_DIR` -> `storage.dir`
+- `CRAWLER.INITIAL_URLS` -> `startUrls`
+- `CRAWLER.INCLUDE_URLS` -> `include`
+- `CRAWLER.EXCLUDE_PATTERNS` -> `exclude`
+- `CRAWLER.ALLOWED_CONTENT_TYPES` -> `allowedContentTypes`
+- `CRAWLER.DOM_ELEMENTS_REMOVE` -> `extract.removeSelectors`
+- `CRAWLER.DYNAMIC_CRAWLING: true` -> `fetcher: 'browser'`
+- `REQUEST_TIMEOUT` / `MAX_REDIRECTS` / `MAX_CONTENT_LENGTH` -> `request.*`
+- `MAX_RETRIES` / `RETRY_STATUS_CODES` / `CRAWL_ERROR_RETRY_DELAY_MS` -> `retry.{max,statusCodes,delay}`
+- `CRAWL_RATE_LIMIT_FALLBACK_DELAY_MS` / `EXIT_ON_RATE_LIMIT` / `EXIT_CODE_RATE_LIMIT` -> `rateLimit.*`
+- `CRAWL_DELAY_MS` -> `crawl.delay`
+- `DATA_FORMATTER.CATEGORISED_PATHS` -> `output.routes`
+- `DATA_FORMATTER.EXCLUDED_PATTERNS` -> `output.exclude`
+
+New in 2.0: `crawl.concurrency`, `crawl.maxDepth`, `crawl.resetOnComplete`, `output.format`, `browser.waitUntil`, `browser.blockResources`, pluggable `fetcher`, and lifecycle hooks. Formatted output is now real JSON by default (1.x wrote `url content` text lines).

package/src/config/browser.js

@@ -0,0 +1,37 @@
+/** @type {readonly ['load', 'domcontentloaded', 'networkidle0', 'networkidle2']} */
+export const BROWSER_WAIT_UNTIL = Object.freeze([
+  'load',
+  'domcontentloaded',
+  'networkidle0',
+  'networkidle2'
+]);
+
+/** Puppeteer resource types Scraply may block to speed up browser fetches. */
+export const BROWSER_BLOCKABLE_RESOURCES = Object.freeze(['image', 'stylesheet', 'font', 'media']);
+
+/** Default blocked types. Stylesheets are excluded — many SPAs need CSS before content renders. */
+export const DEFAULT_BROWSER_BLOCK_RESOURCES = Object.freeze(['image', 'font', 'media']);
+
+/**
+ * @param {import('../index.js').BrowserConfig} browser
+ */
+export const assertBrowserConfig = (browser) => {
+  if (!BROWSER_WAIT_UNTIL.includes(browser?.waitUntil)) {
+    throw new Error(
+      `Invalid browser.waitUntil: ${String(browser?.waitUntil)}. Expected one of: ${BROWSER_WAIT_UNTIL.join(', ')}`
+    );
+  }
+
+  const blockResources = browser?.blockResources;
+  if (!Array.isArray(blockResources)) {
+    throw new Error('Invalid browser.blockResources: expected an array.');
+  }
+
+  for (const type of blockResources) {
+    if (!BROWSER_BLOCKABLE_RESOURCES.includes(type)) {
+      throw new Error(
+        `Invalid browser.blockResources entry: ${String(type)}. Expected one of: ${BROWSER_BLOCKABLE_RESOURCES.join(', ')}`
+      );
+    }
+  }
+};

package/src/config/defaults.js

@@ -0,0 +1,108 @@
+import { DEFAULT_BROWSER_BLOCK_RESOURCES } from './browser.js';
+
+/**
+ * Default Scraply configuration. Every value here can be overridden by the object passed to `createCrawler()` / `scraply()`. Durations are in milliseconds.
+ *
+ * @type {import('../index.js').ScraplyConfig}
+ */
+export const DEFAULT_CONFIG = {
+  // URLs the crawl is seeded with.
+  startUrls: ['https://crawler-test.com/'],
+
+  // Which discovered links are allowed into the queue. Each entry is either an absolute URL prefix (e.g. 'https://site.com/blog') or a RegExp. Empty means "default to startUrls".
+  include: [],
+
+  // Links matching any of these (string prefix or RegExp) are never queued.
+  exclude: [
+    /\.(zip|rar|webp|png|jpg|jpeg|gif|mp3|mp4|pdf|css|js|svg|ico|eot|ttf|woff|woff2|otf|webm|ogg|wav|flac|m4a|mkv|mov|avi|wmv|flv|swf|exe|msi|dmg|iso|bin)$/i
+  ],
+
+  // Only responses whose Content-Type includes one of these are parsed.
+  allowedContentTypes: ['text/html'],
+
+  // 'http' (native fetch), 'browser' (Puppeteer) or a custom Fetcher instance.
+  fetcher: 'http',
+
+  // Options for the built-in Puppeteer fetcher (`fetcher: 'browser'`).
+  browser: {
+    // When page.goto considers navigation finished. Use 'networkidle2' for SPAs that inject links/content after load (e.g. Vue/React apps).
+    waitUntil: 'load',
+
+    // Resource types to abort during fetch (speeds up crawls). Stylesheets are omitted by default because many SPAs need CSS before content renders.
+    blockResources: [...DEFAULT_BROWSER_BLOCK_RESOURCES]
+  },
+
+  // 'silent' | 'error' | 'warn' | 'info' | 'debug'
+  logLevel: 'info',
+
+  storage: {
+    dir: 'dataset'
+  },
+
+  request: {
+    timeout: 10000, // per-request budget (aborts the fetch, including body read)
+    maxRedirects: 5, // redirect hops the HTTP fetcher follows before giving up
+    maxContentLength: 20 * 1024 * 1024, // hard cap on the response body (bytes); 0 disables it
+    userAgent: 'Mozilla/5.0 (compatible; Scraply/2.0; +https://www.npmjs.com/package/scraply)',
+    headers: {} // extra request headers (auth, Accept-Language, cookies, ...) sent by every fetcher
+  },
+
+  retry: {
+    max: 1,
+    statusCodes: [408, 500, 502, 503, 504],
+    delay: 1000
+  },
+
+  rateLimit: {
+    fallbackDelay: 60000,
+    exitOnLimit: true,
+    exitCode: 10
+  },
+
+  crawl: {
+    concurrency: 5,
+    delay: 200, // minimum spacing between requests to the same host
+    maxDepth: Infinity,
+    maxPages: Infinity, // hard cap on successfully crawled pages (counts across resumes)
+    resetOnComplete: true,
+    retryErrors: false // re-queue previously errored URLs on resume so they are retried
+  },
+
+  extract: {
+    removeSelectors: [
+      'script',
+      'noscript',
+      'style',
+      'meta',
+      'link',
+      'svg',
+      'path',
+      'img',
+      'input',
+      'textarea',
+      'embed',
+      'object',
+      'iframe',
+      'nav',
+      'header',
+      'footer',
+      'aside',
+      'button',
+      '[aria-modal]',
+      '[role="dialog"]',
+      '[role="alert"]',
+      '[role="banner"]',
+      '[role="form"]',
+      '[role="navigation"]',
+      '[role="search"]'
+    ]
+  },
+
+  output: {
+    format: 'json', // 'json' | 'jsonl' | 'lines'
+    exclude: [],
+    routes: {
+      'https://crawler-test.com': { '*': 'general.json' }
+    }
+  }
+};

package/src/config/load.js

@@ -0,0 +1,46 @@
+import path from 'node:path';
+import { DEFAULT_CONFIG } from './defaults.js';
+import { assertBrowserConfig } from './browser.js';
+import { normalizeUrl } from '../url/normalize.js';
+
+const isPlainObject = (value) =>
+  value !== null && typeof value === 'object' && !Array.isArray(value) && !(value instanceof RegExp);
+
+const deepMerge = (target, source) => {
+  const merged = { ...target };
+
+  for (const [key, value] of Object.entries(source)) {
+    if (isPlainObject(value) && isPlainObject(target[key])) {
+      merged[key] = deepMerge(target[key], value);
+    } else if (value !== undefined) {
+      merged[key] = value;
+    }
+  }
+
+  return merged;
+};
+
+/**
+ * Merges a user config over the defaults and derives the storage paths.
+ * @param {import('../index.js').ScraplyConfig} [userConfig]
+ * @returns {import('../index.js').ResolvedConfig}
+ */
+export const loadConfig = (userConfig = {}) => {
+  const config = deepMerge(DEFAULT_CONFIG, userConfig);
+
+  const { dir } = config.storage;
+  config.storage.queuePath = path.posix.join(dir, 'queue.json');
+  config.storage.crawledDir = path.posix.join(dir, 'crawled');
+  config.storage.formattedDir = path.posix.join(dir, 'formatted');
+
+  // When no include rules are given, fall back to the start URLs — normalized so
+  // they match the normalized links the crawler actually discovers (forced
+  // HTTPS, no "www.", no trailing slash).
+  if (!config.include?.length) {
+    config.include = config.startUrls.map(normalizeUrl);
+  }
+
+  assertBrowserConfig(config.browser);
+
+  return config;
+};

package/src/core/pipeline.js

@@ -0,0 +1,61 @@
+import { URL } from 'node:url';
+import { delay } from '../util/delay.js';
+
+/**
+ * Drains the queue with a fixed-size worker pool. Requests to the same host are
+ * spaced by `perHostDelay` for politeness, while different hosts run in parallel.
+ * Workers stop when the queue is drained and nothing is in flight, or when
+ * `isStopped()` becomes true.
+ *
+ * @param {Object} deps
+ * @param {import('./queue.js').QueueManager} deps.queue
+ * @param {number} deps.concurrency
+ * @param {number} deps.perHostDelay
+ * @param {(entry: import('./queue.js').QueueEntry) => Promise<void>} deps.processOne
+ * @param {() => boolean} deps.isStopped
+ */
+export const runPipeline = async ({ queue, concurrency, perHostDelay, processOne, isStopped }) => {
+  const lastHostAt = new Map();
+  let active = 0;
+
+  const respectHostDelay = async (url) => {
+    if (perHostDelay <= 0) return;
+
+    let host;
+    try {
+      host = new URL(url).host;
+    } catch {
+      return;
+    }
+
+    const now = Date.now();
+    const scheduled = Math.max(now, (lastHostAt.get(host) ?? 0) + perHostDelay);
+    lastHostAt.set(host, scheduled);
+
+    const wait = scheduled - now;
+    if (wait > 0) await delay(wait);
+  };
+
+  const worker = async () => {
+    while (!isStopped()) {
+      const entry = queue.claimNext();
+
+      if (!entry) {
+        if (active === 0) return; // queue drained and nothing can enqueue more
+        await delay(25);
+        continue;
+      }
+
+      active++;
+      try {
+        await respectHostDelay(entry.url);
+        await processOne(entry);
+      } finally {
+        active--;
+      }
+    }
+  };
+
+  const workers = Array.from({ length: Math.max(concurrency, 1) }, () => worker());
+  await Promise.all(workers);
+};