scraply 1.0.4 → 1.0.6

package/.github/workflows/npm-publish.yml

@@ -2,10 +2,8 @@ name: Publish to NPM
 
 on:
   push:
-    branches:
-      - main
     tags:
-      - 'v*'
+      - 'v*' # Trigger only when a version tag is pushed
 
 jobs:
   publish:
@@ -13,12 +11,12 @@ jobs:
 
     steps:
       - name: Check out the repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Node.js
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v3
         with:
-          node-version: '20.17'
+          node-version: '20'
           registry-url: 'https://registry.npmjs.org/'
 
       - name: Install dependencies

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.4",
+  "version": "1.0.6",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/readme.md

@@ -82,8 +82,19 @@ DATA_FORMATTER: {
   CATEGORISED_PATHS: {
     'https://crawler-test.com': {
       'mobile': 'mobile.json',
-      'fallback': 'general.json'
+      '*': 'general.json'
     },
   },
+  HARD_CODED_LINKS: [
+    {
+      file_name: 'hc-links.json',
+      data: [
+        {
+          "url": "https://custom-link.com",
+          "content": "That's a custom link content, you can add as many as you want."
+        },
+      ]
+    }
+  ]
 }
-```
+```

package/src/defaultConfig.js

@@ -48,8 +48,19 @@ export const DEFAULT_CONFIG = {
     CATEGORISED_PATHS: {
       'https://crawler-test.com': {
         'mobile': 'mobile.json',
-        'fallback': 'general.json'
+        '*': 'general.json'
       },
     },
+    HARD_CODED_LINKS: [
+      {
+        file_name: 'hc-links.json',
+        data: [
+          {
+            "url": "https://custom-link.com",
+            "content": "That's a custom link content, you can add as many as you want."
+          },
+        ]
+      }
+    ]
   }
 };

package/src/scraply.js

@@ -98,8 +98,8 @@ const start = async () => {
   console.log(`${totalSavedURLs} total saved URLs to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
   // Save hardcoded extra links to files.
-  await saveHardcodedExtraLinks();
-  console.log(`Hardcoded extra links saved to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
+  const totalHardcodedLinks = await saveHardcodedExtraLinks();
+  console.log(`${totalHardcodedLinks} Hardcoded extra links saved to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
   // Error reporting: Save into CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH the URLs that had any error: Save the url, the referrer, status code and error!
   const errorData = errorUrls.map(entry => {

package/src/utils/format/formatData.js

@@ -1,5 +1,5 @@
-import fs from 'fs';
-import path from 'path';
+import fs from 'node:fs';
+import path from 'node:path';
 
 export const formatData = (entry) => {
   if (entry.file && entry.error === null) {
@@ -9,7 +9,15 @@ export const formatData = (entry) => {
       const isExcluded = CONFIG.DATA_FORMATTER.EXCLUDED_PATTERNS.some(pattern => new RegExp(pattern).test(entry.url));
   
       if (!isExcluded) {
-        const categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.[pathname.split('/')[1]] || CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.fallback;
+        // Check for the specific category path
+        const pathSegments = pathname.split('/');
+        let categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.[pathSegments[1]];
+        
+        // If no specific category path is found, use the "*" fallback
+        if (!categorisedPath) {
+          categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.['*'];
+        }
+
         if (categorisedPath) {
           return path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
         }
@@ -39,18 +47,12 @@ export const saveSortedFormattedJSON = (filePath, data) => {
 };
 
 export const saveHardcodedExtraLinks = async () => {
-  const data = {
-    file_name: 'cs-links.json',
-    data: [
-        {
-          "url": "https://elemn.to/ai",
-          "content": "🧠 AI - How to save time - This page provides valuable insights on how to leverage AI tools for optimizing workflows and saving time across various tasks."
-        },
-    ],
-  };
-
-  const filePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, data.file_name);
-  saveSortedFormattedJSON(filePath, data.data);
-
-  return data.data.length;
+  const hardcodedLinks = CONFIG.DATA_FORMATTER.HARD_CODED_LINKS;
+
+  for (const link of hardcodedLinks) {
+    const filePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, link.file_name);
+    saveSortedFormattedJSON(filePath, link.data);
+  }
+
+  return hardcodedLinks.reduce((acc, link) => acc + link.data.length, 0); // Total number of links saved
 };