scraply 1.0.12 → 1.0.14

package/.github/workflows/npm-publish.yml

@@ -16,11 +16,11 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: '20'
+          node-version: '22'
           registry-url: 'https://registry.npmjs.org/'
 
       - name: Install dependencies
-        run: npm install
+        run: npm ci
 
       - name: Publish to npm
         run: npm publish

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.12",
+  "version": "1.0.14",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/src/scraply.js

@@ -2,8 +2,7 @@ import { loadConfig } from './loadConfig.js';
 import { normalizeURL } from './utils/crawl/url/normalize.js';
 import { loadJSON, saveQueue, deleteDataFiles, deleteUntrackedFiles } from './utils/crawl/fileOperations.js';
 import { processURL } from './utils/crawl/url/processor.js';
-import { formatData, saveSortedFormattedJSON, saveHardcodedExtraLinks } from './utils/format/formatData.js';
-import path from 'node:path';
+import { formatData, saveSortedFormattedJSON } from './utils/format/formatData.js';
 
 let urlData = [];
 let urlMetadata = {};
@@ -104,16 +103,6 @@ const start = async () => {
   };
   console.log(`${totalSavedURLs} total saved URLs to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
-  // Save hardcoded extra links to files.
-  const totalHardcodedLinks = await saveHardcodedExtraLinks();
-  console.log(`${totalHardcodedLinks} Hardcoded extra links saved to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
-
-  // Track the files generated for hardcoded links with full paths.
-  CONFIG.DATA_FORMATTER.HARD_CODED_LINKS.forEach(link => {
-    const hardcodedFilePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, link.file_name);
-    generatedFiles.add(hardcodedFilePath); // Add the full path to the set
-  });
-
   // Error reporting: Save into CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH the URLs that had any error: Save the url, the referrer, status code and error!
   const errorData = errorUrls.map(entry => {
     return { url: entry.url, status: entry.status, error: entry.error };

package/src/utils/format/formatData.js

@@ -9,17 +9,22 @@ export const formatData = (entry) => {
       const isExcluded = CONFIG.DATA_FORMATTER.EXCLUDED_PATTERNS.some(pattern => new RegExp(pattern).test(entry.url));
   
       if (!isExcluded) {
-        // Check for the specific category path
-        const pathSegments = pathname.split('/');
-        let categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.[pathSegments[1]];
-        
-        // If no specific category path is found, use the "*" fallback
-        if (!categorisedPath) {
-          categorisedPath = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin]?.['*'];
-        }
-
-        if (categorisedPath) {
-          return path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
+        const pathSegments = pathname.split('/').filter(Boolean); // filter out empty segments
+        const categorisedPaths = CONFIG.DATA_FORMATTER.CATEGORISED_PATHS[url.origin];
+
+        if (categorisedPaths) {
+          // Try to match the full path segments, reducing specificity step by step
+          let categorisedPath = null;
+
+          for (let i = pathSegments.length; i >= 1; i--) {
+            const pathKey = pathSegments.slice(0, i).join('/');
+            categorisedPath = categorisedPaths[pathKey];
+            if (categorisedPath) break;
+          }
+
+          // Fallback to wildcard match ('*') if no specific path is found
+          if (!categorisedPath) categorisedPath = categorisedPaths['*'];
+          if (categorisedPath) return path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
         }
       }
     } catch (e) {
@@ -45,14 +50,3 @@ export const saveSortedFormattedJSON = (filePath, data) => {
   const sortedData = sortData(data, 'url'); // ensure data is sorted before saving
   return fs.writeFileSync(filePath, JSON.stringify(sortedData, null, 2), 'utf8');
 };
-
-export const saveHardcodedExtraLinks = async () => {
-  const hardcodedLinks = CONFIG.DATA_FORMATTER.HARD_CODED_LINKS;
-
-  for (const link of hardcodedLinks) {
-    const filePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, link.file_name);
-    saveSortedFormattedJSON(filePath, link.data);
-  }
-
-  return hardcodedLinks.reduce((acc, link) => acc + link.data.length, 0); // Total number of links saved
-};