website-scrap-engine 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/downloader/main.d.ts.map +1 -1
- package/lib/downloader/main.js +4 -1
- package/lib/downloader/main.js.map +1 -1
- package/lib/options.d.ts.map +1 -1
- package/lib/options.js +7 -1
- package/lib/options.js.map +1 -1
- package/package.json +1 -1
- package/src/downloader/main.ts +4 -1
- package/src/options.ts +7 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/downloader/main.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAG7B,OAAO,KAAK,EAAC,eAAe,EAAE,qBAAqB,EAAC,MAAM,eAAe,CAAC;AAE1E,OAAO,KAAK,EAAC,WAAW,EAAW,MAAM,gBAAgB,CAAC;AAI1D,OAAO,KAAK,EAAC,eAAe,EAAE,kBAAkB,EAAC,MAAM,YAAY,CAAC;AACpE,OAAO,EAAC,oBAAoB,EAAC,MAAM,6BAA6B,CAAC;AAEjE,8BAAsB,kBAAmB,YAAW,kBAAkB;IAkBvC,aAAa,EAAE,MAAM;IAjBlD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IACjD,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACvF,QAAQ,CAAC,EAAE,eAAe,CAAC;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,oBAAoB,CAAC;IACjC,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,CAAqB;IACxD,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAqB;IACpD,QAAQ,CAAC,IAAI,EAAE,eAAe,CAK5B;IACF,WAAW,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,GAAG,IAAI,CAAa;IAE/D,SAAS,aAAoB,aAAa,EAAE,MAAM,EAChD,eAAe,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;
|
|
1
|
+
{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/downloader/main.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAG7B,OAAO,KAAK,EAAC,eAAe,EAAE,qBAAqB,EAAC,MAAM,eAAe,CAAC;AAE1E,OAAO,KAAK,EAAC,WAAW,EAAW,MAAM,gBAAgB,CAAC;AAI1D,OAAO,KAAK,EAAC,eAAe,EAAE,kBAAkB,EAAC,MAAM,YAAY,CAAC;AACpE,OAAO,EAAC,oBAAoB,EAAC,MAAM,6BAA6B,CAAC;AAEjE,8BAAsB,kBAAmB,YAAW,kBAAkB;IAkBvC,aAAa,EAAE,MAAM;IAjBlD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC,eAAe,CAAC,CAAC;IACjD,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACvF,QAAQ,CAAC,EAAE,eAAe,CAAC;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,oBAAoB,CAAC;IACjC,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,CAAqB;IACxD,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAqB;IACpD,QAAQ,CAAC,IAAI,EAAE,eAAe,CAK5B;IACF,WAAW,EAAE,UAAU,CAAC,OAAO,WAAW,CAAC,GAAG,IAAI,CAAa;IAE/D,SAAS,aAAoB,aAAa,EAAE,MAAM,EAChD,eAAe,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE;IAoB9E,SAAS,CAAC,aAAa,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhE,IAAI,OAAO,IAAI,eAAe,CAK7B;IAED,IAAI,QAAQ,IAAI,oBAAoB,CAKnC;IAED,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,IAAI,WAAW,CAAC,cAAc,EAAE,MAAM,EAErC;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,IAAI,YAAY,IAAI,MAAM,CAEzB;IAEK,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BzD,SAAS,CAAC,qBAAqB,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,GAAG,IAAI;IAwBjE,QAAQ,CAAC,0BAA0B,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;IAE9E,oBAAoB,CAAC,GAAG,EAAE,WAAW,GAAG,OAAO,GAAG,IAAI;IAStD,WAAW,CAAC,GAAG,EAAE,KAAK,GAAG,OAAO,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,GAAG,IAAI;IAapF,IAAI,eAAe,IAAI,MAAM,CAE5B;IAED,KAAK,IAAI,IAAI;IAcb,IAAI,IAAI,IAAI;IAOZ,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAIjB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAM/B"}
|
package/lib/downloader/main.js
CHANGED
|
@@ -19,11 +19,14 @@ export class AbstractDownloader {
|
|
|
19
19
|
this.adjustTimer = undefined;
|
|
20
20
|
this._asyncOptions = importDefaultFromPath(pathToOptions);
|
|
21
21
|
this._overrideOptions = overrideOptions;
|
|
22
|
-
|
|
22
|
+
// A safeguard here, concurrency is set later
|
|
23
|
+
this.queue = new PQueue({ concurrency: 2 });
|
|
23
24
|
this._isInit = false;
|
|
24
25
|
this._initOptions = this._asyncOptions.then(options => {
|
|
25
26
|
options = mergeOverrideOptions(options, this._overrideOptions);
|
|
26
27
|
this._options = options;
|
|
28
|
+
// https://github.com/website-local/website-scrap-engine/issues/1113
|
|
29
|
+
this.queue.concurrency = options.concurrency;
|
|
27
30
|
this._pipeline = new PipelineExecutorImpl(options, options.req, options);
|
|
28
31
|
options.configureLogger(options.localRoot, options.logSubDir || '');
|
|
29
32
|
return this._internalInit(options).then(() => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/downloader/main.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,OAAO,GAAG,MAAM,OAAO,CAAC;AAExB,OAAO,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAEnD,OAAO,EAAC,iBAAiB,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAC,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAC,qBAAqB,EAAC,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAC,oBAAoB,EAAC,MAAM,6BAA6B,CAAC;AAEjE,MAAM,OAAgB,kBAAkB;IAkBtC,YAA6B,aAAqB,EAChD,eAA4E;QADjD,kBAAa,GAAb,aAAa,CAAQ;QAVzC,kBAAa,GAAgB,IAAI,GAAG,EAAU,CAAC;QAC/C,cAAS,GAAgB,IAAI,GAAG,EAAU,CAAC;QAC3C,SAAI,GAAoB;YAC/B,kBAAkB,EAAE,CAAC;YACrB,gBAAgB,EAAE,CAAC;YACnB,eAAe,EAAE,CAAC;YAClB,oBAAoB,EAAE,CAAC;SACxB,CAAC;QACF,gBAAW,GAA0C,SAAS,CAAC;QAI7D,IAAI,CAAC,aAAa,GAAG,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC1D,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,IAAI,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/downloader/main.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAE7B,OAAO,GAAG,MAAM,OAAO,CAAC;AAExB,OAAO,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAEnD,OAAO,EAAC,iBAAiB,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAC,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAC,qBAAqB,EAAC,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAC,oBAAoB,EAAC,MAAM,6BAA6B,CAAC;AAEjE,MAAM,OAAgB,kBAAkB;IAkBtC,YAA6B,aAAqB,EAChD,eAA4E;QADjD,kBAAa,GAAb,aAAa,CAAQ;QAVzC,kBAAa,GAAgB,IAAI,GAAG,EAAU,CAAC;QAC/C,cAAS,GAAgB,IAAI,GAAG,EAAU,CAAC;QAC3C,SAAI,GAAoB;YAC/B,kBAAkB,EAAE,CAAC;YACrB,gBAAgB,EAAE,CAAC;YACnB,eAAe,EAAE,CAAC;YAClB,oBAAoB,EAAE,CAAC;SACxB,CAAC;QACF,gBAAW,GAA0C,SAAS,CAAC;QAI7D,IAAI,CAAC,aAAa,GAAG,qBAAqB,CAAC,aAAa,CAAC,CAAC;QAC1D,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,6CAA6C;QAC7C,IAAI,CAAC,KAAK,GAAG,IAAI,MAAM,CAAC,EAAC,WAAW,EAAE,CAAC,EAAC,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;YACpD,OAAO,GAAG,oBAAoB,CAAC,OAAO,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC/D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;YACxB,oEAAoE;YACpE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;YAC7C,IAAI,CAAC,SAAS,GAAG,IAAI,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACzE,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;YACpE,OAAO,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE;gBAC3C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,6DAA6D;IACnD,aAAa,CAAC,OAAwB;QAC9C,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,IAAI,OAAO;QACT,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,QAAQ;QACV,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,SAAS,CAAC;QACxB,CAAC;QACD,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC7D,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;IAChC,CAAC;IAED,IAAI,WAAW,CAAC,cAAsB;QACpC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,cAAc,CAAC;IAC1C,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,MAAgB;QACvC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACpB,8CAA8C;YAC9C,MAAM,IAAI,CAAC,YAAY,CAAC;QAC1B,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACpC,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,IAAI,GAAG,GAAkB,MAAM,CAAC,CAAC,CAAC,CAAC;YACnC,GAAG,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YACnD,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,MAAM,IAAI,GAAwB,MAAM,QAAQ,CAAC,kBAAkB,CACjE,GAAG,EAAE,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,IAAI,CAAC,GAAoB,MAAM,QAAQ,CAAC,cAAc,CACpD,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EACjB,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;YACzC,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,CAAC,GAAG,MAAM,QAAQ,CAAC,qBAAqB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YACxD,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,IAAI,CAAC,CAAC,CAAC,6BAA6B,EAAE,CAAC;gBACrC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAES,qBAAqB,CAAC,GAAgB;;QAC9C,8BAA8B;QAC9B,IAAI,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACtC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/D,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,GAAW,CAAC;QAChB,MAAM,GAAG,GAAQ,CAAC,CAAA,MAAC,GAAgB,aAAhB,GAAG,uBAAH,GAAG,CAAe,GAAG,0CAAE,KAAK,EAAE,KAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5E,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,CAAC;YACxC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,QAAQ,GAAa,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAClD,qBAAqB;QACrB,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAChD,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IAID,oBAAoB,CAAC,GAAgB;QACnC,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAC;YAC5C,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,WAAW,CAAC,GAA2B,EAAE,KAAa,EAAE,QAAqB;;QAC3E,uCAAuC;QACvC,IAAI,GAAG,IAAK,GAAuB,CAAC,IAAI,KAAK,WAAW;YACtD,CAAA,MAAC,GAAiB,aAAjB,GAAG,uBAAH,GAAG,CAAgB,QAAQ,0CAAE,UAAU,MAAK,GAAG,EAAE,CAAC;YACnD,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACvE,CAAC;aAAM,IAAI,GAAG,EAAE,CAAC;YACf,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAChF,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAGD,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;IACjC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE;YAC1B,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,qBAAqB,KAAK,UAAU,EAAE,CAAC;gBAC7D,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACrB,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAClC,CAAC;gBACD,IAAI,CAAC,WAAW,GAAG,WAAW,CAC5B,GAAG,EAAE,eAAC,OAAA,MAAA,MAAA,IAAI,CAAC,OAAO,EAAC,qBAAqB,mDAAG,IAAI,CAAC,CAAA,EAAA,EAChD,IAAI,CAAC,OAAO,CAAC,uBAAuB,IAAI,KAAK,CAAC,CAAC;YACnD,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,OAAO;;QACX,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,CAAA,MAAA,IAAI,CAAC,QAAQ,0CAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA,CAAC;IACpD,CAAC;CAEF"}
|
package/lib/options.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../src/options.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAE,aAAa,EAAe,YAAY,EAAC,MAAM,KAAK,CAAC;AAEhF,OAAO,KAAK,EAAC,gBAAgB,EAAE,YAAY,EAAC,MAAM,eAAe,CAAC;AAElE,OAAO,KAAK,EAAC,mBAAmB,EAAE,cAAc,EAAC,MAAM,uBAAuB,CAAC;AAM/E,OAAO,EAAC,eAAe,EAAC,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,EAAC,uBAAuB,EAAC,MAAM,YAAY,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,kBACf,SAAQ,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAE,IAAI,CAAC;IACvD,oBAAoB,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;IACtD,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;;;OAKG;IACH,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;IAEjD;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAEjC;;OAEG;IACH,YAAY,CAAC,EAAE,uBAAuB,CAAC;IAEvC;;OAEG;IACH,gBAAgB,CAAC,EAAE,uBAAuB,CAAC;IAE3C;;OAEG;IACH,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE7B;;;;;;;;;OASG;IACH,GAAG,CAAC,EAAE,cAAc,CAAC;IAErB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,IAAI,EAAE,kBAAkB,CAAC;IAEzB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;;OAMG;IACH,4BAA4B,CAAC,EAAE,OAAO,CAAC;CACxC;AAED,MAAM,WAAW,eAAgB,SAAQ,qBAAqB,EAAE,mBAAmB;IACjF;;;;OAIG;IACH,GAAG,EAAE,cAAc,CAAC;IAEpB;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,qBAAqB,CAAC,EAAE,CAAC,UAAU,EAAE,kBAAkB,KAAK,IAAI,CAAC;IAEjE;;OAEG;IACH,eAAe,EAAE,OAAO,eAAe,CAAC;CACzC;AAED,MAAM,MAAM,aAAa,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC,GAAG;IAC1D,kBAAkB,EAAE,OAAO,CAAC;CAC7B,CAAC;AAyBF;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,EAAE,aAqEhC,CAAC;AAyBF,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,eAAe,CAAC,GAAG,eAAe,CAkD1E;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,eAAe,GAAG,eAAe,CAiB9E;AAED,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,eAAe,GAAG,CAAC,MAAM,eAAe,CAAC,EAClD,eAAe,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG,eAAe,
|
|
1
|
+
{"version":3,"file":"options.d.ts","sourceRoot":"","sources":["../src/options.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,YAAY,EAAE,aAAa,EAAe,YAAY,EAAC,MAAM,KAAK,CAAC;AAEhF,OAAO,KAAK,EAAC,gBAAgB,EAAE,YAAY,EAAC,MAAM,eAAe,CAAC;AAElE,OAAO,KAAK,EAAC,mBAAmB,EAAE,cAAc,EAAC,MAAM,uBAAuB,CAAC;AAM/E,OAAO,EAAC,eAAe,EAAC,MAAM,2BAA2B,CAAC;AAC1D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,uBAAuB,CAAC;AAE9D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,EAAC,uBAAuB,EAAC,MAAM,YAAY,CAAC;AAExD;;GAEG;AACH,MAAM,WAAW,kBACf,SAAQ,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAE,IAAI,CAAC;IACvD,oBAAoB,CAAC,EAAE,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;IACtD,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;;;OAKG;IACH,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;IAEjD;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;IAEjC;;OAEG;IACH,YAAY,CAAC,EAAE,uBAAuB,CAAC;IAEvC;;OAEG;IACH,gBAAgB,CAAC,EAAE,uBAAuB,CAAC;IAE3C;;OAEG;IACH,OAAO,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE7B;;;;;;;;;OASG;IACH,GAAG,CAAC,EAAE,cAAc,CAAC;IAErB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IAEtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,IAAI,EAAE,kBAAkB,CAAC;IAEzB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAE/B;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAEhC;;;;;;OAMG;IACH,4BAA4B,CAAC,EAAE,OAAO,CAAC;CACxC;AAED,MAAM,WAAW,eAAgB,SAAQ,qBAAqB,EAAE,mBAAmB;IACjF;;;;OAIG;IACH,GAAG,EAAE,cAAc,CAAC;IAEpB;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,qBAAqB,CAAC,EAAE,CAAC,UAAU,EAAE,kBAAkB,KAAK,IAAI,CAAC;IAEjE;;OAEG;IACH,eAAe,EAAE,OAAO,eAAe,CAAC;CACzC;AAED,MAAM,MAAM,aAAa,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC,GAAG;IAC1D,kBAAkB,EAAE,OAAO,CAAC;CAC7B,CAAC;AAyBF;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,EAAE,aAqEhC,CAAC;AAyBF,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,eAAe,CAAC,GAAG,eAAe,CAkD1E;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,eAAe,GAAG,eAAe,CAiB9E;AAED,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,eAAe,GAAG,CAAC,MAAM,eAAe,CAAC,EAClD,eAAe,CAAC,EAAE,OAAO,CAAC,qBAAqB,CAAC,GAAG,eAAe,CAmBnE"}
|
package/lib/options.js
CHANGED
|
@@ -206,7 +206,13 @@ export function mergeOverrideOptions(options, overrideOptions) {
|
|
|
206
206
|
}
|
|
207
207
|
if (opt.req && overrideOptions.req) {
|
|
208
208
|
const options = got.defaults.options;
|
|
209
|
-
|
|
209
|
+
const mergedOptions = new Options(opt.req, overrideOptions.req, options);
|
|
210
|
+
// New versions of got removed `mergeOptions`
|
|
211
|
+
// Instances of `Options` can not be reused, or it will result in memory leak
|
|
212
|
+
// Will try to find a better way as there is no public api for this
|
|
213
|
+
// See https://github.com/website-local/website-scrap-engine/issues/1112
|
|
214
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
215
|
+
overrideOptions.req = mergedOptions._internals;
|
|
210
216
|
}
|
|
211
217
|
return checkDownloadOptions(Object.assign(opt, overrideOptions));
|
|
212
218
|
}
|
package/lib/options.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"options.js","sourceRoot":"","sources":["../src/options.ts"],"names":[],"mappings":"AACA,OAAO,GAAG,EAAE,EAAC,OAAO,EAAC,MAAM,KAAK,CAAC;AAEjC,OAAO,EAAC,cAAc,EAAC,MAAM,eAAe,CAAC;AAE7C,oCAAoC;AACpC,OAAO,EAAC,eAAe,EAAC,MAAM,mCAAmC,CAAC;AAClE,OAAO,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AACzC,oCAAoC;AACpC,OAAO,EAAC,MAAM,EAAC,MAAM,oCAAoC,CAAC;AAC1D,OAAO,EAAC,eAAe,EAAC,MAAM,2BAA2B,CAAC;AAE1D,OAAO,EAAC,UAAU,EAAC,MAAM,WAAW,CAAC;AAiKrC,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,MAAM,eAAe,GAAgB,IAAI,GAAG,CAAC;IAC3C,0CAA0C;IAC1C,WAAW;IACX,6CAA6C;IAC7C,YAAY;IACZ,mCAAmC;IACnC,YAAY;IACZ,wCAAwC;IACxC,cAAc;IACd,+DAA+D;IAC/D,OAAO;IACP,kDAAkD;IAClD,WAAW;IACX,0BAA0B;IAC1B,aAAa;IACb,wBAAwB;IACxB,WAAW;IACX,4BAA4B;IAC5B,WAAW;CACZ,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAkB,CAAC,WAAwB,EAAU,EAAE;IACpF,MAAM,EAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,EAAC,GAAG,WAAW,CAAC;IAE7D,IAAI,YAAY,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC;QACrC,GAAqB,CAAC,kBAAkB,GAAG,IAAI,CAAC;QACjD,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,CAAC;QACL,GAAqB,CAAC,kBAAkB,GAAG,KAAK,CAAC;IACpD,CAAC;IAED,MAAM,SAAS,GAAY,GAAG,CAAC,OAAO;QACpC,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC5B,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YACnD,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,GAAG,CAAC,IAAI;QAC3B,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/B,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5C,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAc,CAAC,CAAC,CAAC;IAC7C,MAAM,aAAa,GAAwB,YAAY,CAAC,WAAW;QACjE,GAAG,CAAC,QAAQ;QACZ,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC7D,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,aAAa;QAChD,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,CAAC,EAAE,CAAC;QAE3D,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,WAAW;YACpC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;YACpD,KAAK,CAAC,KAAK,CAAC,wBAAwB,EAClC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAG,GAAoB,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,EAC5D,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,KAAK,GAAW,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;IAC5E,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,KAAK,IAAI,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,KAAK,GAAG,eAAe,EAAE,CAAC;QAC5B,KAAK,GAAG,eAAe,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC;IACzD,CAAC;IACD,wBAAwB;IACxB,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW;QAC1B,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAClD,mBAAmB;QACnB,KAAK,IAAI,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC;QACrC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO;YACtB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACtC,IAAI,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7B,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,UAAU,IAAI,IAAI,CAAC;YACrB,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;gBACvB,UAAU,IAAI,CAAC,CAAC;gBAChB,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;oBACnB,UAAU,GAAG,CAAC,CAAC;gBACjB,CAAC;gBACD,IAAI,YAAY,CAAC,aAAa,EAAE,CAAC;oBAC/B,IAAI,UAAU,IAAI,YAAY,CAAC,aAAa,EAAE,CAAC;wBAC7C,KAAK,GAAG,UAAU,CAAC;oBACrB,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,KAAK,GAAG,UAAU,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,IAAI,CAAC,CAAC;IACX,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,cAAc,GAAoB;IACtC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,EAAE;IACX,WAAW,EAAE,EAAE;IACf,eAAe;IACf,cAAc;IACd,kBAAkB,EAAE,EAAE;IACtB,QAAQ,EAAE,EAAE;IACZ,mBAAmB;IACnB,QAAQ,EAAE,EAAiC;IAC3C,YAAY,EAAE,EAAE;IAChB,SAAS,EAAE,EAAE;IACb,QAAQ,EAAE,CAAC;IACX,IAAI,EAAE;QACJ,oBAAoB,EAAE,SAAS;KAChC;IACD,oBAAoB,EAAE,EAAE;IACxB,qBAAqB,EAAE,EAAE;IACzB,GAAG,EAAE,EAAE;IACP,UAAU,EAAE,EAAE;IACd,sBAAsB,EAAE,IAAI;CAC7B,CAAC;AAEF,MAAM,UAAU,sBAAsB,CACpC,OAAuD;IACvD,MAAM,MAAM,GAAoB,UAAU,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IACpE,+CAA+C;IAC/C,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,CAAC,WAAW,GAAG,EAAE,CAAC;IAC1B,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC;IACxB,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,eAAe,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,CAAC,CAAC,cAAc,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC;IAC/B,CAAC;IACD,IAAI,CAAC,CAAC,sBAAsB,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,CAAC,oBAAoB,GAAG,IAAI,CAAC;IACzC,CAAC;IACD,IAAI,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACnE,MAAM,CAAC,GAAG,CAAC,OAAO,GAAG;YACnB,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,IAAI;YACb,aAAa,EAAE,IAAI;YACnB,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,MAAM;SAChB,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/D,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG;YACjB,KAAK,EAAE,EAAE;YACT,aAAa,EAAE,KAAK;YACpB,cAAc,EAAE,kBAAkB;SACnC,CAAC;IACJ,CAAC;SAAM,IAAI,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG;YACjB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK;YACvB,aAAa,EAAE,KAAK;YACpB,cAAc,EAAE,kBAAkB;SACnC,CAAC;IACJ,CAAC;SAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,GAAG,kBAAkB,CAAC;IACvD,CAAC;IACD,IAAI,OAAO,CAAC,uBAAuB;QACjC,OAAO,CAAC,uBAAuB,GAAG,CAAC;QACnC,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAC;QACjC,OAAO,CAAC,qBAAqB,GAAG,MAAM,CAAC;IACzC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,OAAwB;;IAC3D,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,SAAS,CAAC,mBAAmB,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,SAAS,CAAC,uBAAuB,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,MAAA,OAAO,CAAC,YAAY,0CAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,OAAO,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAClD,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACtD,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,sBAAsB,CAAC,OAAO,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,OAAkD,EAClD,eAAgD;IAChD,MAAM,GAAG,GAAoB,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IACjF,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,IAAI,eAAe,CAAC,IAAI,EAAE,CAAC;QACrC,eAAe,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,GAAG,CAAC,GAAG,IAAI,eAAe,CAAC,GAAG,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QACrC,
|
|
1
|
+
{"version":3,"file":"options.js","sourceRoot":"","sources":["../src/options.ts"],"names":[],"mappings":"AACA,OAAO,GAAG,EAAE,EAAC,OAAO,EAAC,MAAM,KAAK,CAAC;AAEjC,OAAO,EAAC,cAAc,EAAC,MAAM,eAAe,CAAC;AAE7C,oCAAoC;AACpC,OAAO,EAAC,eAAe,EAAC,MAAM,mCAAmC,CAAC;AAClE,OAAO,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AACzC,oCAAoC;AACpC,OAAO,EAAC,MAAM,EAAC,MAAM,oCAAoC,CAAC;AAC1D,OAAO,EAAC,eAAe,EAAC,MAAM,2BAA2B,CAAC;AAE1D,OAAO,EAAC,UAAU,EAAC,MAAM,WAAW,CAAC;AAiKrC,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,MAAM,eAAe,GAAgB,IAAI,GAAG,CAAC;IAC3C,0CAA0C;IAC1C,WAAW;IACX,6CAA6C;IAC7C,YAAY;IACZ,mCAAmC;IACnC,YAAY;IACZ,wCAAwC;IACxC,cAAc;IACd,+DAA+D;IAC/D,OAAO;IACP,kDAAkD;IAClD,WAAW;IACX,0BAA0B;IAC1B,aAAa;IACb,wBAAwB;IACxB,WAAW;IACX,4BAA4B;IAC5B,WAAW;CACZ,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAkB,CAAC,WAAwB,EAAU,EAAE;IACpF,MAAM,EAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,EAAC,GAAG,WAAW,CAAC;IAE7D,IAAI,YAAY,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC;QACrC,GAAqB,CAAC,kBAAkB,GAAG,IAAI,CAAC;QACjD,OAAO,CAAC,CAAC;IACX,CAAC;SAAM,CAAC;QACL,GAAqB,CAAC,kBAAkB,GAAG,KAAK,CAAC;IACpD,CAAC;IAED,MAAM,SAAS,GAAY,GAAG,CAAC,OAAO;QACpC,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC5B,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YACnD,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,GAAG,CAAC,IAAI;QAC3B,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/B,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5C,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAc,CAAC,CAAC,CAAC;IAC7C,MAAM,aAAa,GAAwB,YAAY,CAAC,WAAW;QACjE,GAAG,CAAC,QAAQ;QACZ,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAC7D,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,aAAa;QAChD,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,CAAC,EAAE,CAAC;QAE3D,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,KAAK,WAAW;YACpC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;YACpD,KAAK,CAAC,KAAK,CAAC,wBAAwB,EAClC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAG,GAAoB,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,EAC5D,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,KAAK,GAAW,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;IAC5E,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,KAAK,IAAI,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,KAAK,GAAG,eAAe,EAAE,CAAC;QAC5B,KAAK,GAAG,eAAe,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC;IACzD,CAAC;IACD,wBAAwB;IACxB,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW;QAC1B,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAClD,mBAAmB;QACnB,KAAK,IAAI,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC;QACrC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO;YACtB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACtC,IAAI,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;YAC/D,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7B,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,UAAU,IAAI,IAAI,CAAC;YACrB,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;gBACvB,UAAU,IAAI,CAAC,CAAC;gBAChB,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;oBACnB,UAAU,GAAG,CAAC,CAAC;gBACjB,CAAC;gBACD,IAAI,YAAY,CAAC,aAAa,EAAE,CAAC;oBAC/B,IAAI,UAAU,IAAI,YAAY,CAAC,aAAa,EAAE,CAAC;wBAC7C,KAAK,GAAG,UAAU,CAAC;oBACrB,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,KAAK,GAAG,UAAU,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,IAAI,CAAC,CAAC;IACX,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,cAAc,GAAoB;IACtC,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,EAAE;IACX,WAAW,EAAE,EAAE;IACf,eAAe;IACf,cAAc;IACd,kBAAkB,EAAE,EAAE;IACtB,QAAQ,EAAE,EAAE;IACZ,mBAAmB;IACnB,QAAQ,EAAE,EAAiC;IAC3C,YAAY,EAAE,EAAE;IAChB,SAAS,EAAE,EAAE;IACb,QAAQ,EAAE,CAAC;IACX,IAAI,EAAE;QACJ,oBAAoB,EAAE,SAAS;KAChC;IACD,oBAAoB,EAAE,EAAE;IACxB,qBAAqB,EAAE,EAAE;IACzB,GAAG,EAAE,EAAE;IACP,UAAU,EAAE,EAAE;IACd,sBAAsB,EAAE,IAAI;CAC7B,CAAC;AAEF,MAAM,UAAU,sBAAsB,CACpC,OAAuD;IACvD,MAAM,MAAM,GAAoB,UAAU,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IACpE,+CAA+C;IAC/C,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,CAAC,WAAW,GAAG,EAAE,CAAC;IAC1B,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC;IACxB,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,eAAe,CAAC,CAAC;IACnD,CAAC;IACD,IAAI,CAAC,CAAC,cAAc,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,CAAC,YAAY,GAAG,EAAE,CAAC;IAC/B,CAAC;IACD,IAAI,CAAC,CAAC,sBAAsB,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,CAAC,oBAAoB,GAAG,IAAI,CAAC;IACzC,CAAC;IACD,IAAI,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACnE,MAAM,CAAC,GAAG,CAAC,OAAO,GAAG;YACnB,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE,IAAI;YACb,aAAa,EAAE,IAAI;YACnB,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,IAAI;YACV,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,MAAM;SAChB,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/D,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG;YACjB,KAAK,EAAE,EAAE;YACT,aAAa,EAAE,KAAK;YACpB,cAAc,EAAE,kBAAkB;SACnC,CAAC;IACJ,CAAC;SAAM,IAAI,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG;YACjB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK;YACvB,aAAa,EAAE,KAAK;YACpB,cAAc,EAAE,kBAAkB;SACnC,CAAC;IACJ,CAAC;SAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,GAAG,kBAAkB,CAAC;IACvD,CAAC;IACD,IAAI,OAAO,CAAC,uBAAuB;QACjC,OAAO,CAAC,uBAAuB,GAAG,CAAC;QACnC,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAC;QACjC,OAAO,CAAC,qBAAqB,GAAG,MAAM,CAAC;IACzC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,OAAwB;;IAC3D,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,SAAS,CAAC,mBAAmB,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QACvB,MAAM,IAAI,SAAS,CAAC,uBAAuB,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,MAAA,OAAO,CAAC,YAAY,0CAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,OAAO,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAClD,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;QACtD,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,sBAAsB,CAAC,OAAO,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,OAAkD,EAClD,eAAgD;IAChD,MAAM,GAAG,GAAoB,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;IACjF,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,IAAI,eAAe,CAAC,IAAI,EAAE,CAAC;QACrC,eAAe,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC;IACvE,CAAC;IACD,IAAI,GAAG,CAAC,GAAG,IAAI,eAAe,CAAC,GAAG,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QACrC,MAAM,aAAa,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzE,6CAA6C;QAC7C,6EAA6E;QAC7E,mEAAmE;QACnE,wEAAwE;QACxE,8DAA8D;QAC9D,eAAe,CAAC,GAAG,GAAI,aAAqB,CAAC,UAAU,CAAC;IAC1D,CAAC;IACD,OAAO,oBAAoB,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,CAAC;AACnE,CAAC"}
|
package/package.json
CHANGED
package/src/downloader/main.ts
CHANGED
|
@@ -32,11 +32,14 @@ export abstract class AbstractDownloader implements DownloaderWithMeta {
|
|
|
32
32
|
overrideOptions?: Partial<StaticDownloadOptions> & { pathToWorker?: string }) {
|
|
33
33
|
this._asyncOptions = importDefaultFromPath(pathToOptions);
|
|
34
34
|
this._overrideOptions = overrideOptions;
|
|
35
|
-
|
|
35
|
+
// A safeguard here, concurrency is set later
|
|
36
|
+
this.queue = new PQueue({concurrency: 2});
|
|
36
37
|
this._isInit = false;
|
|
37
38
|
this._initOptions = this._asyncOptions.then(options => {
|
|
38
39
|
options = mergeOverrideOptions(options, this._overrideOptions);
|
|
39
40
|
this._options = options;
|
|
41
|
+
// https://github.com/website-local/website-scrap-engine/issues/1113
|
|
42
|
+
this.queue.concurrency = options.concurrency;
|
|
40
43
|
this._pipeline = new PipelineExecutorImpl(options, options.req, options);
|
|
41
44
|
options.configureLogger(options.localRoot, options.logSubDir || '');
|
|
42
45
|
return this._internalInit(options).then(() => {
|
package/src/options.ts
CHANGED
|
@@ -378,7 +378,13 @@ export function mergeOverrideOptions(
|
|
|
378
378
|
}
|
|
379
379
|
if (opt.req && overrideOptions.req) {
|
|
380
380
|
const options = got.defaults.options;
|
|
381
|
-
|
|
381
|
+
const mergedOptions = new Options(opt.req, overrideOptions.req, options);
|
|
382
|
+
// New versions of got removed `mergeOptions`
|
|
383
|
+
// Instances of `Options` can not be reused, or it will result in memory leak
|
|
384
|
+
// Will try to find a better way as there is no public api for this
|
|
385
|
+
// See https://github.com/website-local/website-scrap-engine/issues/1112
|
|
386
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
387
|
+
overrideOptions.req = (mergedOptions as any)._internals;
|
|
382
388
|
}
|
|
383
389
|
return checkDownloadOptions(Object.assign(opt, overrideOptions));
|
|
384
390
|
}
|