scraply 1.0.15 → 1.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/package.json +1 -1
  2. package/src/loadConfig.js +4 -4
  3. package/src/utils/crawl/fileOperations.js +3 -3
  4. package/src/utils/format/formatData.js +1 -1
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "scraply",
3
3
  "description": "A simple, configurable and functional content scraper",
4
- "version": "1.0.15",
4
+ "version": "1.0.16",
5
5
  "main": "src/scraply.js",
6
6
  "type": "module",
7
7
  "scripts": {
package/src/loadConfig.js CHANGED
@@ -16,10 +16,10 @@ export function loadConfig(userConfig = {}) {
16
16
  const config = deepMerge(DEFAULT_CONFIG, userConfig);
17
17
 
18
18
  // Dynamically construct paths using MAIN_DIR
19
- config.CRAWLER.QUEUE_PATH = path.join(config.MAIN_DIR, 'queue.json');
20
- config.CRAWLER.CRAWLED_PATH = path.join(config.MAIN_DIR, 'crawled');
21
- config.DATA_FORMATTER.FORMATTED_PATH = path.join(config.MAIN_DIR, 'formatted');
22
- config.DATA_FORMATTER.ERROR_REPORT_PATH = path.join(config.MAIN_DIR, 'error-report.json');
19
+ config.CRAWLER.QUEUE_PATH = path.posix.join(config.MAIN_DIR, 'queue.json');
20
+ config.CRAWLER.CRAWLED_PATH = path.posix.join(config.MAIN_DIR, 'crawled');
21
+ config.DATA_FORMATTER.FORMATTED_PATH = path.posix.join(config.MAIN_DIR, 'formatted');
22
+ config.DATA_FORMATTER.ERROR_REPORT_PATH = path.posix.join(config.MAIN_DIR, 'error-report.json');
23
23
 
24
24
  // If INCLUDE_URLS is not specified, set it to INITIAL_URLS by default
25
25
  if (!config.CRAWLER.INCLUDE_URLS || config.CRAWLER.INCLUDE_URLS.length === 0) {
package/src/utils/crawl/fileOperations.js CHANGED
@@ -11,7 +11,7 @@ export const saveJSON = (filePath, data) => {
11
11
 
12
12
  export const saveDataset = (data, fileNumber) => {
13
13
  if (!fs.existsSync(CONFIG.CRAWLER.CRAWLED_PATH)) fs.mkdirSync(CONFIG.CRAWLER.CRAWLED_PATH, { recursive: true });
14
- const filename = path.join(CONFIG.CRAWLER.CRAWLED_PATH, `${fileNumber}.json`);
14
+ const filename = path.posix.join(CONFIG.CRAWLER.CRAWLED_PATH, `${fileNumber}.json`);
15
15
  fs.writeFileSync(filename, JSON.stringify(data, null, 2), 'utf8');
16
16
  return filename;
17
17
  };
@@ -22,7 +22,7 @@ export const deleteDataFiles = (filePath) => {
22
22
  if (fs.existsSync(filePath)) {
23
23
  if (fs.lstatSync(filePath).isDirectory()) {
24
24
  fs.readdirSync(filePath).forEach((file) => {
25
- const currentPath = path.join(filePath, file);
25
+ const currentPath = path.posix.join(filePath, file);
26
26
  if (fs.lstatSync(currentPath).isDirectory()) {
27
27
  deleteDataFiles(currentPath);
28
28
  } else {
@@ -39,7 +39,7 @@ export const deleteDataFiles = (filePath) => {
39
39
  export const deleteUntrackedFiles = (folderPath, trackedFiles) => {
40
40
  if (fs.existsSync(folderPath)) {
41
41
  fs.readdirSync(folderPath).forEach((file) => {
42
- const currentPath = path.join(folderPath, file);
42
+ const currentPath = path.posix.join(folderPath, file);
43
43
  if (fs.lstatSync(currentPath).isDirectory()) {
44
44
  deleteUntrackedFiles(currentPath, trackedFiles);
45
45
  } else if (!trackedFiles.has(currentPath)) {
package/src/utils/format/formatData.js CHANGED
@@ -24,7 +24,7 @@ export const formatData = (entry) => {
24
24
 
25
25
  // Fallback to wildcard match ('*') if no specific path is found
26
26
  if (!categorisedPath) categorisedPath = categorisedPaths['*'];
27
- if (categorisedPath) return path.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
27
+ if (categorisedPath) return path.posix.join(CONFIG.DATA_FORMATTER.FORMATTED_PATH, categorisedPath); // Return the path where the data should be saved.
28
28
  }
29
29
  }
30
30
  } catch (e) {