scraply 1.0.13 → 1.0.14

package/.github/workflows/npm-publish.yml

@@ -16,11 +16,11 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v3
         with:
-          node-version: '20'
+          node-version: '22'
           registry-url: 'https://registry.npmjs.org/'
 
       - name: Install dependencies
-        run: npm install
+        run: npm ci
 
       - name: Publish to npm
         run: npm publish

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "scraply",
   "description": "A simple, configurable and functional content scraper",
-  "version": "1.0.13",
+  "version": "1.0.14",
   "main": "src/scraply.js",
   "type": "module",
   "scripts": {

package/src/scraply.js

@@ -2,8 +2,7 @@ import { loadConfig } from './loadConfig.js';
 import { normalizeURL } from './utils/crawl/url/normalize.js';
 import { loadJSON, saveQueue, deleteDataFiles, deleteUntrackedFiles } from './utils/crawl/fileOperations.js';
 import { processURL } from './utils/crawl/url/processor.js';
-import { formatData, saveSortedFormattedJSON, saveHardcodedExtraLinks } from './utils/format/formatData.js';
-import path from 'node:path';
+import { formatData, saveSortedFormattedJSON } from './utils/format/formatData.js';
 
 let urlData = [];
 let urlMetadata = {};
@@ -104,16 +103,6 @@ const start = async () => {
   };
   console.log(`${totalSavedURLs} total saved URLs to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
 
-  // Save hardcoded extra links to files.
-  const totalHardcodedLinks = await saveHardcodedExtraLinks();
-  console.log(`${totalHardcodedLinks} Hardcoded extra links saved to ${CONFIG.DATA_FORMATTER.FORMATTED_PATH}`);
-
-  // Track the files generated for hardcoded links with full paths.
-  CONFIG.DATA_FORMATTER.HARD_CODED_LINKS.forEach(link => {
-    const hardcodedFilePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, link.file_name);
-    generatedFiles.add(hardcodedFilePath); // Add the full path to the set
-  });
-
   // Error reporting: Save into CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH the URLs that had any error: Save the url, the referrer, status code and error!
   const errorData = errorUrls.map(entry => {
     return { url: entry.url, status: entry.status, error: entry.error };

package/src/utils/format/formatData.js

@@ -50,14 +50,3 @@ export const saveSortedFormattedJSON = (filePath, data) => {
   const sortedData = sortData(data, 'url'); // ensure data is sorted before saving
   return fs.writeFileSync(filePath, JSON.stringify(sortedData, null, 2), 'utf8');
 };
-
-export const saveHardcodedExtraLinks = async () => {
-  const hardcodedLinks = CONFIG.DATA_FORMATTER.HARD_CODED_LINKS;
-
-  for (const link of hardcodedLinks) {
-    const filePath = path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, link.file_name);
-    saveSortedFormattedJSON(filePath, link.data);
-  }
-
-  return hardcodedLinks.reduce((acc, link) => acc + link.data.length, 0); // Total number of links saved
-};