website-scrap-engine 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/lib/io.js +2 -2
- package/lib/io.js.map +1 -1
- package/package.json +8 -8
- package/src/io.ts +4 -4
- package/tsconfig.json +0 -1
package/CHANGELOG.md
CHANGED
package/lib/io.js
CHANGED
|
@@ -6,14 +6,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.writeFile = exports.mkdirRetry = void 0;
|
|
7
7
|
const fs_1 = __importDefault(require("fs"));
|
|
8
8
|
const path_1 = require("path");
|
|
9
|
-
const mkdirp_1 =
|
|
9
|
+
const mkdirp_1 = require("mkdirp");
|
|
10
10
|
const logger_1 = require("./logger/logger");
|
|
11
11
|
const mkdirRetry = async (dir, retry = 3) => {
|
|
12
12
|
let error;
|
|
13
13
|
for (let i = 0; i < retry; i++) {
|
|
14
14
|
error = undefined;
|
|
15
15
|
try {
|
|
16
|
-
await (0, mkdirp_1.
|
|
16
|
+
await (0, mkdirp_1.mkdirp)(dir);
|
|
17
17
|
}
|
|
18
18
|
catch (e) {
|
|
19
19
|
error = e;
|
package/lib/io.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"io.js","sourceRoot":"","sources":["../src/io.ts"],"names":[],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"io.js","sourceRoot":"","sources":["../src/io.ts"],"names":[],"mappings":";;;;;;AACA,4CAAoB;AACpB,+BAA6B;AAC7B,mCAA8B;AAE9B,4CAA2E;AAEpE,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,KAAK,GAAG,CAAC,EAAiB,EAAE;IACxE,IAAI,KAAqB,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE;QAC9B,KAAK,GAAG,SAAS,CAAC;QAClB,IAAI;YACF,MAAM,IAAA,eAAM,EAAC,GAAG,CAAC,CAAC;SACnB;QAAC,OAAO,CAAC,EAAE;YACV,KAAK,GAAG,CAAC,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,EAAE;gBACT,cAAW,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;aACxD;iBAAM;gBACL,cAAW,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;aACxD;YACD,SAAS;SACV;QACD,KAAK,GAAG,SAAS,CAAC;QAClB,OAAO;KACR;IACD,IAAI,KAAK,EAAE;QACT,MAAM,KAAK,CAAC;KACb;AACH,CAAC,CAAC;AArBW,QAAA,UAAU,cAqBrB;AAEK,MAAM,SAAS,GAAG,KAAK,EAC5B,QAAgB,EAChB,IAAkB,EAClB,QAA0B,EAC1B,KAAqB,EACrB,KAAqB,EACN,EAAE;IACjB,MAAM,GAAG,GAAW,IAAA,cAAO,EAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;QACvB,MAAM,IAAA,kBAAU,EAAC,GAAG,CAAC,CAAC;KACvB;IACD,IAAI,QAA6B,CAAC;IAClC,IAAI,OAAqC,CAAC;IAC1C,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;QAC5B,QAAQ,GAAG,IAAI,CAAC;QAChB,OAAO,GAAG,EAAC,QAAQ,EAAC,CAAC;KACtB;SAAM,IAAI,IAAI,YAAY,WAAW,EAAE;QACtC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;SAAM,IAAI,IAAI,YAAY,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;QAC9D,QAAQ,GAAG,IAAI,CAAC;KACjB;SAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACnC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;KACvE;SAAM;QACL,oBAAoB;QACpB,MAAM,IAAI,SAAS,CAAC,6BAA6B,CAAC,CAAC;KACpD;IACD,IAAI,OAAO,EAAE;QACX,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;KAC1D;SAAM;QACL,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;KACjD;IACD,qBAAqB;IACrB,IAAI,KAAK,EAAE;QACT,IAAI,CAAC,KAAK,EAAE;YACV,KAAK,GAAG,KAAK,CAAC;SACf;QACD,IAAI;YACF,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACV,cAAW,CAAC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,EAAE,CAAC,CAAC,CAAC;SACpD;KACF;AACH,CAAC,CAAC;AA1CW,QAAA,SAAS,aA0CpB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "website-scrap-engine",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Configurable website scraper in typescript",
|
|
5
5
|
"main": "lib",
|
|
6
6
|
"types": "lib",
|
|
@@ -24,22 +24,22 @@
|
|
|
24
24
|
"cheerio": "^1.0.0-rc.12",
|
|
25
25
|
"css-url-parser": "^1.1.3",
|
|
26
26
|
"got": "^11.8.6",
|
|
27
|
-
"log4js": "^6.
|
|
28
|
-
"mkdirp": "^
|
|
27
|
+
"log4js": "^6.9.1",
|
|
28
|
+
"mkdirp": "^3.0.0",
|
|
29
29
|
"p-queue": "^6.6.2",
|
|
30
30
|
"srcset": "^4.0.0",
|
|
31
31
|
"urijs": "^1.19.11"
|
|
32
32
|
},
|
|
33
33
|
"devDependencies": {
|
|
34
34
|
"@types/jest": "^27.5.2",
|
|
35
|
-
"@types/node": "^18.
|
|
35
|
+
"@types/node": "^18.15.13",
|
|
36
36
|
"@types/urijs": "^1.19.19",
|
|
37
|
-
"@typescript-eslint/eslint-plugin": "^5.
|
|
38
|
-
"@typescript-eslint/parser": "^5.
|
|
39
|
-
"eslint": "^8.
|
|
37
|
+
"@typescript-eslint/eslint-plugin": "^5.59.0",
|
|
38
|
+
"@typescript-eslint/parser": "^5.59.0",
|
|
39
|
+
"eslint": "^8.38.0",
|
|
40
40
|
"jest": "^27.5.1",
|
|
41
41
|
"ts-jest": "^27.1.5",
|
|
42
|
-
"typescript": "^
|
|
42
|
+
"typescript": "^5.0.4"
|
|
43
43
|
},
|
|
44
44
|
"files": [
|
|
45
45
|
".editorconfig",
|
package/src/io.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import fs from 'fs';
|
|
2
1
|
import type {ObjectEncodingOptions} from 'fs';
|
|
2
|
+
import fs from 'fs';
|
|
3
3
|
import {dirname} from 'path';
|
|
4
|
-
import
|
|
4
|
+
import {mkdirp} from 'mkdirp';
|
|
5
5
|
import type {ResourceBody, ResourceEncoding} from './resource';
|
|
6
|
-
import {
|
|
6
|
+
import {error as errorLogger, mkdir as mkdirLogger} from './logger/logger';
|
|
7
7
|
|
|
8
8
|
export const mkdirRetry = async (dir: string, retry = 3): Promise<void> => {
|
|
9
9
|
let error: unknown | void;
|
|
10
10
|
for (let i = 0; i < retry; i++) {
|
|
11
11
|
error = undefined;
|
|
12
12
|
try {
|
|
13
|
-
await
|
|
13
|
+
await mkdirp(dir);
|
|
14
14
|
} catch (e) {
|
|
15
15
|
error = e;
|
|
16
16
|
if (i > 0) {
|