scraply 1.0.5 → 1.0.7

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/readme.md

@@ -82,8 +82,19 @@ DATA_FORMATTER: {
   CATEGORISED_PATHS: {
     'https://crawler-test.com': {
       'mobile': 'mobile.json',
-      'fallback': 'general.json'
+      '*': 'general.json'
     },
   },
+  HARD_CODED_LINKS: [
+    {
+      file_name: 'hc-links.json',
+      data: [
+        {
+          "url": "https://custom-link.com",
+          "content": "That's a custom link content, you can add as many as you want."
+        },
+      ]
+    }
+  ]
 }
-```
+```

package/src/defaultConfig.js

@@ -48,8 +48,19 @@ export const DEFAULT_CONFIG = {
     CATEGORISED_PATHS: {
       'https://crawler-test.com': {
         'mobile': 'mobile.json',
-        'fallback': 'general.json'
+        '*': 'general.json'
       },
     },
+    HARD_CODED_LINKS: [
+      {
+        file_name: 'hc-links.json',
+        data: [
+          {
+            "url": "https://custom-link.com",
+            "content": "That's a custom link content, you can add as many as you want."
+          },
+        ]
+      }
+    ]
   }
 };

package/src/scraply.js

@@ -98,8 +98,8 @@ const start = async () => {
   console.log(`${totalSavedURLs} total saved URLs to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
   // Save hardcoded extra links to files.
-  await saveHardcodedExtraLinks();
-  console.log(`Hardcoded extra links saved to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
+  const totalHardcodedLinks = await saveHardcodedExtraLinks();
+  console.log(`${totalHardcodedLinks} Hardcoded extra links saved to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
   // Error reporting: Save into CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH the URLs that had any error: Save the url, the referrer, status code and error!
   const errorData = errorUrls.map(entry => {

package/src/utils/crawl/url/handlers.js

@@ -19,7 +19,28 @@ export const shouldRetry = (error) => {
 const shouldIncludeURL = (url) => {
   try {
     const urlObj = new URL(url);
-    return CONFIG.CRAWLER.INCLUDE_URLS.some(pattern => new RegExp(pattern).test(urlObj.toString())) && !CONFIG.CRAWLER.EXCLUDE_PATTERNS.some(pattern => new RegExp(pattern).test(urlObj.pathname));
+
+    // Check if the URL matches any include pattern
+    const isIncluded = CONFIG.CRAWLER.INCLUDE_URLS.some(pattern => {
+      if (typeof pattern === 'string') {
+        return urlObj.toString().includes(pattern); // Check if the URL contains the string pattern
+      } else if (pattern instanceof RegExp) {
+        return pattern.test(urlObj.toString()); // Check if the URL matches the RegExp pattern
+      }
+    });
+
+    if (!isIncluded) return false; // This URL is not included
+
+    // Check if the URL matches any exclude pattern
+    const isExcluded = CONFIG.CRAWLER.EXCLUDE_PATTERNS.some(pattern => {
+      if (typeof pattern === 'string') {
+        return urlObj.pathname.includes(pattern); // Check if the URL pathname contains the string pattern
+      } else if (pattern instanceof RegExp) {
+        return pattern.test(urlObj.pathname); // Check if the URL pathname matches the RegExp pattern
+      }
+    });
+
+    return !isExcluded; // This URL is included and not excluded
   } catch (error) {
     return false;
   }

package/src/utils/format/formatData.js

@@ -1,5 +1,5 @@
-import fs from 'fs';
-import path from 'path';
+import fs from 'node:fs';
+import path from 'node:path';
 
 export const formatData = (entry) => {
   if (entry.file && entry.error === null) {
@@ -9,7 +9,15 @@ export const formatData = (entry) => {
       const isExcluded = CONFIG.DATA_FORMATTER.EXCLUDED_PATTERNS.some(pattern => new RegExp(pattern).test(entry.url));
   
       if (!isExcluded) {
-        const categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.[pathname.split('/')[1]] || CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.fallback;
+        // Check for the specific category path
+        const pathSegments = pathname.split('/');
+        let categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.[pathSegments[1]];
+        
+        // If no specific category path is found, use the "*" fallback
+        if (!categorisedPath) {
+          categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.['*'];
+        }
+
         if (categorisedPath) {
           return path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
         }
@@ -39,18 +47,12 @@ export const saveSortedFormattedJSON = (filePath, data) => {
 };
 
 export const saveHardcodedExtraLinks = async () => {
-  const data = {
-    file_name: 'cs-links.json',
-    data: [
-        {
-          "url": "https://elemn.to/ai",
-          "content": "🧠 AI - How to save time - This page provides valuable insights on how to leverage AI tools for optimizing workflows and saving time across various tasks."
-        },
-    ],
-  };
-
-  const filePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, data.file_name);
-  saveSortedFormattedJSON(filePath, data.data);
-
-  return data.data.length;
+  const hardcodedLinks = CONFIG.DATA_FORMATTER.HARD_CODED_LINKS;
+
+  for (const link of hardcodedLinks) {
+    const filePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, link.file_name);
+    saveSortedFormattedJSON(filePath, link.data);
+  }
+
+  return hardcodedLinks.reduce((acc, link) => acc + link.data.length, 0); // Total number of links saved
 };