scraply 1.0.6 → 1.0.8

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.6",
+  "version": "1.0.8",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/src/utils/crawl/url/handlers.js

@@ -18,9 +18,24 @@ export const shouldRetry = (error) => {
 
 const shouldIncludeURL = (url) => {
   try {
-    const urlObj = new URL(url);
-    return CONFIG.CRAWLER.INCLUDE_URLS.some(pattern => new RegExp(pattern).test(urlObj.toString())) && !CONFIG.CRAWLER.EXCLUDE_PATTERNS.some(pattern => new RegExp(pattern).test(urlObj.pathname));
+    const { INITIAL_URLS, INCLUDE_URLS, EXCLUDE_PATTERNS } = CONFIG.CRAWLER;
+
+    if (INITIAL_URLS.includes(url)) return true;
+
+    // Pre-compile string patterns into regular expressions for both include and exclude patterns
+    const compiledExcludePatterns = EXCLUDE_PATTERNS.map(pattern => 
+      typeof pattern === 'string' ? new RegExp(pattern) : pattern
+    );
+    const compiledIncludePatterns = INCLUDE_URLS.map(pattern => 
+      typeof pattern === 'string' ? new RegExp(pattern) : pattern
+    );
+
+    if (compiledExcludePatterns.some(pattern => pattern.test(url))) return false;
+    if (compiledIncludePatterns.some(pattern => pattern.test(url))) return true;
+
+    return false; // If the URL doesn't match any include patterns, exclude it.
   } catch (error) {
+    console.error(`Error processing URL: ${url}`, error);
     return false;
   }
 };