scraply 1.0.1 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/scraply.js +12 -8
package/package.json
CHANGED
package/src/scraply.js
CHANGED
|
@@ -4,16 +4,11 @@ import { loadJSON, saveQueue, deleteDataFiles } from './utils/crawl/fileOperatio
|
|
|
4
4
|
import { processURL } from './utils/crawl/url/processor.js';
|
|
5
5
|
import { formatData, saveSortedFormattedJSON, saveHardcodedExtraLinks } from './utils/format/formatData.js';
|
|
6
6
|
|
|
7
|
-
const userConfig = {};
|
|
8
|
-
|
|
9
|
-
// Load and merge the configuration
|
|
10
|
-
const CONFIG = loadConfig(userConfig);
|
|
11
|
-
global.CONFIG = CONFIG;
|
|
12
|
-
|
|
13
7
|
let urlData = [];
|
|
14
8
|
let urlMetadata = {};
|
|
9
|
+
let CONFIG = {};
|
|
15
10
|
|
|
16
|
-
|
|
11
|
+
const initializeCrawler = () => {
|
|
17
12
|
urlData = loadJSON(CONFIG.CRAWLER.QUEUE_PATH);
|
|
18
13
|
|
|
19
14
|
if (urlData.length === 0) { // If the queue is empty, start fresh with the initial URLs.
|
|
@@ -44,7 +39,7 @@ export const initializeCrawler = () => {
|
|
|
44
39
|
}
|
|
45
40
|
};
|
|
46
41
|
|
|
47
|
-
|
|
42
|
+
const crawler = async () => {
|
|
48
43
|
console.log(`STARTING CRAWLER
|
|
49
44
|
- Initial URLs: ${CONFIG.CRAWLER.INITIAL_URLS}
|
|
50
45
|
- Include URLs: ${CONFIG.CRAWLER.INCLUDE_URLS}
|
|
@@ -115,3 +110,12 @@ export const scraply = async () => {
|
|
|
115
110
|
|
|
116
111
|
console.log(`Errors: ${errorData.length} -> ${CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH}.`);
|
|
117
112
|
};
|
|
113
|
+
|
|
114
|
+
// Main function to be exported and used
|
|
115
|
+
export const scraply = async (userConfig = {}) => {
|
|
116
|
+
CONFIG = loadConfig(userConfig);
|
|
117
|
+
global.CONFIG = CONFIG;
|
|
118
|
+
|
|
119
|
+
initializeCrawler();
|
|
120
|
+
await crawler();
|
|
121
|
+
};
|