scraply 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -3
- package/readme.md +4 -0
- package/src/scraply.js +2 -5
package/package.json
CHANGED
|
@@ -1,17 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scraply",
|
|
3
3
|
"description": "A simple, configurable and functional content scraper",
|
|
4
|
-
"version": "1.0.
|
|
4
|
+
"version": "1.0.1",
|
|
5
5
|
"main": "src/scraply.js",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"scripts": {
|
|
8
8
|
"start": "node ."
|
|
9
9
|
},
|
|
10
|
-
"keywords": [
|
|
10
|
+
"keywords": [
|
|
11
|
+
"crawler",
|
|
12
|
+
"scraper"
|
|
13
|
+
],
|
|
11
14
|
"author": "Pau Serrat Gutiérrez",
|
|
12
15
|
"dependencies": {
|
|
13
16
|
"axios": "^1.7.7",
|
|
14
17
|
"cheerio": "^1.0.0",
|
|
15
18
|
"he": "^1.2.0"
|
|
16
19
|
}
|
|
17
|
-
}
|
|
20
|
+
}
|
package/readme.md
CHANGED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
Scraply is a customizable and efficient web crawler and data scraper for Node.js, designed to handle various web crawling needs with ease. You can define the URLs to crawl, configure patterns to include/exclude, and format the output data in JSON. Scraply is built to be flexible, with user-configurable settings and dynamic paths.
|
|
2
|
+
|
|
3
|
+
Installation
|
|
4
|
+
`npm install scraply`
|
package/src/scraply.js
CHANGED
|
@@ -13,7 +13,7 @@ global.CONFIG = CONFIG;
|
|
|
13
13
|
let urlData = [];
|
|
14
14
|
let urlMetadata = {};
|
|
15
15
|
|
|
16
|
-
const initializeCrawler = () => {
|
|
16
|
+
export const initializeCrawler = () => {
|
|
17
17
|
urlData = loadJSON(CONFIG.CRAWLER.QUEUE_PATH);
|
|
18
18
|
|
|
19
19
|
if (urlData.length === 0) { // If the queue is empty, start fresh with the initial URLs.
|
|
@@ -44,7 +44,7 @@ const initializeCrawler = () => {
|
|
|
44
44
|
}
|
|
45
45
|
};
|
|
46
46
|
|
|
47
|
-
const
|
|
47
|
+
export const scraply = async () => {
|
|
48
48
|
console.log(`STARTING CRAWLER
|
|
49
49
|
- Initial URLs: ${CONFIG.CRAWLER.INITIAL_URLS}
|
|
50
50
|
- Include URLs: ${CONFIG.CRAWLER.INCLUDE_URLS}
|
|
@@ -115,6 +115,3 @@ const app = async () => {
|
|
|
115
115
|
|
|
116
116
|
console.log(`Errors: ${errorData.length} -> ${CONFIG.DATA_FORMATTER.ERROR_REPORT_PATH}.`);
|
|
117
117
|
};
|
|
118
|
-
|
|
119
|
-
initializeCrawler();
|
|
120
|
-
await app();
|