tardis-dev 18.0.0 → 18.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/combine.js CHANGED
@@ -1,4 +1,4 @@
1
- import { isManagedRealTimeIterator, mergeRealTime } from "./realtimeiterator.js";
1
+ import { closeIterator, isManagedRealTimeIterator, mergeRealTime } from "./realtimeiterator.js";
2
2
  const DATE_MAX = new Date(8640000000000000);
3
3
  async function nextWithIndex(iterator, index) {
4
4
  if ('offsetMS' in iterator) {
@@ -91,10 +91,7 @@ async function* combineHistorical(iterators) {
91
91
  } while (aliveIteratorsCount > 0);
92
92
  }
93
93
  finally {
94
- for (let iterator of iterators) {
95
- ;
96
- iterator.return();
97
- }
94
+ await Promise.all(iterators.map((iterator) => closeIterator('stream' in iterator ? iterator.stream : iterator)));
98
95
  }
99
96
  }
100
97
  //# sourceMappingURL=combine.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"combine.js","sourceRoot":"","sources":["../src/combine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAShF,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,gBAAgB,CAAC,CAAA;AAU3C,KAAK,UAAU,aAAa,CAC1B,QAA+G,EAC/G,KAAa;IAEb,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;QAE3C,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,OAAO,QAAQ,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAA;YAE9G,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACnB,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,kBAAkB,EAAE,GAAG,QAAQ,CAAC,CAAA;YAC7G,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM;YACN,KAAK;SACN,CAAA;IACH,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAEpC,OAAO;YACL,MAAM;YACN,KAAK;SACN,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAkC,EAAE,OAAmC;IAC/F,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,MAAM,CAAA;IACf,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACxB,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,CAAA;IACtE,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,CAAA;IAEpE,IAAI,gBAAgB,GAAG,eAAe,EAAE,CAAC;QACvC,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,IAAI,gBAAgB,KAAK,eAAe,EAAE,CAAC;QACzC,MAAM,4BAA4B,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC,CAAA;QAChF,MAAM,2BAA2B,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC,CAAA;QAE9E,IAAI,4BAA4B,GAAG,2BAA2B,EAAE,CAAC;YAC/D,OAAO,OAAO,CAAA;QAChB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,kEAAkE;AAClE,+CAA+C;AAC/C,MAAM,UAAU,OAAO,CAA4B,GAAG,gBAAmB;IACvE,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC,MAAM,CAAA;QACvB,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC,CAAC,CAAA;IACF,IAAI,yBAAyB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,OAAO,aAAa,CAAC,SAAS,CAA8C,CAAA;IAC9E,CAAC;IAED,OAAO,iBAAiB,CAAC,gBAAgB,CAA8C,CAAA;AACzF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,iBAAiB,CAC/B,SAAoH;IAEpH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,kCAAkC;QAClC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;QAC/D,IAAI,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAA;QACxC,GAAG,CAAC;YACF,+CAA+C;YAC/C,6FAA6F;YAE7F,yCAAyC;YACzC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;YACjE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,YAAY,CAAA;YAEtC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,mBAAmB,EAAE,CAAA;gBAErB,yEAAyE;gBACzE,0EAA0E;gBAC1E,uBAAuB;gBAEvB,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG;oBACtB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE;wBACL,cAAc,EAAE,QAAQ;qBACzB;iBACF,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,+EAA+E;gBAC/E,MAAM,MAAM,CAAC,KAAK,CAAA;gBAClB,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;YAC/D,CAAC;QACH,CAAC,QAAQ,mBAAmB,GAAG,CAAC,EAAC;IACnC,CAAC;YAAS,CAAC;QACT,KAAK,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC/B,CAAC;YAAC,QAAgB,CAAC,MAAM,EAAE,CAAA;QAC7B,CAAC;IACH,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"combine.js","sourceRoot":"","sources":["../src/combine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAS/F,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,gBAAgB,CAAC,CAAA;AAU3C,KAAK,UAAU,aAAa,CAC1B,QAA+G,EAC/G,KAAa;IAEb,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;QAE3C,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,OAAO,QAAQ,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAA;YAE9G,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACnB,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,kBAAkB,EAAE,GAAG,QAAQ,CAAC,CAAA;YAC7G,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM;YACN,KAAK;SACN,CAAA;IACH,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAEpC,OAAO;YACL,MAAM;YACN,KAAK;SACN,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAkC,EAAE,OAAmC;IAC/F,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,MAAM,CAAA;IACf,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACxB,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,CAAA;IACtE,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,CAAA;IAEpE,IAAI,gBAAgB,GAAG,eAAe,EAAE,CAAC;QACvC,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,IAAI,gBAAgB,KAAK,eAAe,EAAE,CAAC;QACzC,MAAM,4BAA4B,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC,CAAA;QAChF,MAAM,2BAA2B,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC,CAAA;QAE9E,IAAI,4BAA4B,GAAG,2BAA2B,EAAE,CAAC;YAC/D,OAAO,OAAO,CAAA;QAChB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,kEAAkE;AAClE,+CAA+C;AAC/C,MAAM,UAAU,OAAO,CAA4B,GAAG,gBAAmB;IACvE,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC,MAAM,CAAA;QACvB,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC,CAAC,CAAA;IACF,IAAI,yBAAyB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,OAAO,aAAa,CAAC,SAAS,CAA8C,CAAA;IAC9E,CAAC;IAED,OAAO,iBAAiB,CAAC,gBAAgB,CAA8C,CAAA;AACzF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,iBAAiB,CAC/B,SAAoH;IAEpH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,kCAAkC;QAClC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;QAC/D,IAAI,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAA;QACxC,GAAG,CAAC;YACF,+CAA+C;YAC/C,6FAA6F;YAE7F,yCAAyC;YACzC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;YACjE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,YAAY,CAAA;YAEtC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,mBAAmB,EAAE,CAAA;gBAErB,yEAAyE;gBACzE,0EAA0E;gBAC1E,uBAAuB;gBAEvB,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG;oBACtB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE;wBACL,cAAc,EAAE,QAAQ;qBACzB;iBACF,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,+EAA+E;gBAC/E,MAAM,MAAM,CAAC,KAAK,CAAA;gBAClB,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;YAC/D,CAAC;QACH,CAAC,QAAQ,mBAAmB,GAAG,CAAC,EAAC;IACnC,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;IAClH,CAAC;AACH,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"downloaddatasets.d.ts","sourceRoot":"","sources":["../src/downloaddatasets.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAGlD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAQrC,wBAAsB,gBAAgB,CAAC,uBAAuB,EAAE,uBAAuB,iBAyEtF;AAuDD,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,UAE5C;AA6BD,KAAK,kBAAkB,GAAG;IACxB,QAAQ,EAAE,QAAQ,CAAA;IAClB,QAAQ,EAAE,WAAW,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,IAAI,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,QAAQ,CAAA;IAClB,SAAS,EAAE,WAAW,EAAE,CAAA;IACxB,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,CAAC,EAAE,KAAK,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,MAAM,CAAA;IACrD,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB,CAAA"}
1
+ {"version":3,"file":"downloaddatasets.d.ts","sourceRoot":"","sources":["../src/downloaddatasets.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAIlD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAQrC,wBAAsB,gBAAgB,CAAC,uBAAuB,EAAE,uBAAuB,iBAyEtF;AAuDD,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,UAE5C;AA6BD,KAAK,kBAAkB,GAAG;IACxB,QAAQ,EAAE,QAAQ,CAAA;IAClB,QAAQ,EAAE,WAAW,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,IAAI,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,QAAQ,CAAA;IAClB,SAAS,EAAE,WAAW,EAAE,CAAA;IACxB,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,CAAC,EAAE,KAAK,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,MAAM,CAAA;IACrD,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB,CAAA"}
@@ -1,6 +1,6 @@
1
1
  import { existsSync } from 'node:fs';
2
- import pMap from 'p-map';
3
2
  import { debug } from "./debug.js";
3
+ import { forEachConcurrent } from "./foreachconcurrent.js";
4
4
  import { addDays, doubleDigit, download, parseAsUTCDate, sequence } from "./handy.js";
5
5
  import { getOptions } from "./options.js";
6
6
  const CONCURRENCY_LIMIT = 20;
@@ -46,8 +46,8 @@ export async function downloadDatasets(downloadDatasetsOptions) {
46
46
  date: startDate
47
47
  }), skipIfExists);
48
48
  // download the rest concurrently up to the CONCURRENCY_LIMIT
49
- await pMap(sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
50
- (offset) => downloadDataSet(getDownloadOptions({
49
+ await forEachConcurrent(sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
50
+ CONCURRENCY_LIMIT, (offset) => downloadDataSet(getDownloadOptions({
51
51
  exchange,
52
52
  symbol,
53
53
  apiKey,
@@ -56,7 +56,7 @@ export async function downloadDatasets(downloadDatasetsOptions) {
56
56
  format,
57
57
  getFilename,
58
58
  date: addDays(startDate, offset)
59
- }), skipIfExists), { concurrency: CONCURRENCY_LIMIT });
59
+ }), skipIfExists));
60
60
  const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000;
61
61
  debug('dataset download finished for %s %s %s from %s to %s, time: %s seconds', exchange, dataType, symbol, from, to, elapsedSeconds);
62
62
  }
@@ -1 +1 @@
1
- {"version":3,"file":"downloaddatasets.js","sourceRoot":"","sources":["../src/downloaddatasets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,IAAI,MAAM,OAAO,CAAA;AACxB,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAGzC,MAAM,iBAAiB,GAAG,EAAE,CAAA;AAC5B,MAAM,0BAA0B,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AACtD,MAAM,oBAAoB,GAAG,YAAY,CAAA;AAEzC,MAAM,OAAO,GAAG,UAAU,EAAE,CAAA;AAE5B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,uBAAgD;IACrF,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,uBAAuB,CAAA;IAC1E,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAA;IAC7G,MAAM,WAAW,GAAG,uBAAuB,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAA;IAClI,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAA;IACpG,MAAM,WAAW,GAAG,uBAAuB,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAA;IAChI,MAAM,YAAY,GAAG,uBAAuB,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB,CAAC,YAAY,CAAA;IAErH,0GAA0G;IAC1G,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;IAEjF,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;QACrC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAC,uBAAuB,CAAC,CAAA;YACrF,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAA;YAC3C,KAAK,CAAC,qDAAqD,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;YAElG,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;gBACzB,mHAAmH;gBACnH,MAAM,eAAe,CACnB,kBAAkB,CAAC;oBACjB,QAAQ;oBACR,MAAM;oBACN,MAAM;oBACN,WAAW;oBACX,QAAQ;oBACR,MAAM;oBACN,WAAW;oBACX,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,gBAAgB,GAAG,CAAC,CAAC;iBAC/C,CAAC,EACF,YAAY,CACb,CAAA;YACH,CAAC;YAED,iHAAiH;YACjH,MAAM,eAAe,CACnB,kBAAkB,CAAC;gBACjB,QAAQ;gBACR,MAAM;gBACN,MAAM;gBACN,WAAW;gBACX,QAAQ;gBACR,MAAM;gBACN,WAAW;gBACX,IAAI,EAAE,SAAS;aAChB,CAAC,EACF,YAAY,CACb,CAAA;YAED,6DAA6D;YAC7D,MAAM,IAAI,CACR,QAAQ,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,gIAAgI;YACnK,CAAC,MAAM,EAAE,EAAE,CACT,eAAe,CACb,kBAAkB,CAAC;gBACjB,QAAQ;gBACR,MAAM;gBACN,MAAM;gBACN,WAAW;gBACX,QAAQ;gBACR,MAAM;gBACN,WAAW;gBACX,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;aACjC,CAAC,EACF,YAAY,CACb,EACH,EAAE,WAAW,EAAE,iBAAiB,EAAE,CACnC,CAAA;YACD,MAAM,cAAc,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC,GAAG,IAAI,CAAA;YAErE,KAAK,CAAC,wEAAwE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,cAAc,CAAC,CAAA;QACvI,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,eAAgC,EAAE,YAAqB;IACpF,IAAI,YAAY,IAAI,UAAU,CAAC,eAAe,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7D,KAAK,CAAC,8CAA8C,EAAE,eAAe,CAAC,YAAY,CAAC,CAAA;QACnF,OAAM;IACR,CAAC;SAAM,CAAC;QACN,OAAO,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAA;IACxC,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,EAC1B,MAAM,EACN,QAAQ,EACR,QAAQ,EACR,IAAI,EACJ,MAAM,EACN,MAAM,EACN,WAAW,EACX,WAAW,EAUZ;IACC,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,EAAE,CAAA;IAClC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAA;IACjD,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAA;IAE1C,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,gBAAgB,IAAI,QAAQ,IAAI,QAAQ,IAAI,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,CAAA;IACpI,MAAM,QAAQ,GAAG,WAAW,CAAC;QAC3B,QAAQ;QACR,IAAI;QACJ,QAAQ;QACR,MAAM;QACN,MAAM;KACP,CAAC,CAAA;IAEF,MAAM,YAAY,GAAG,GAAG,WAAW,IAAI,QAAQ,EAAE,CAAA;IAEjD,OAAO;QACL,GAAG;QACH,YAAY;QACZ,SAAS,EAAE,OAAO,CAAC,UAAU;QAC7B,MAAM;KACP,CAAA;AACH,CAAC;AAID,MAAM,UAAU,mBAAmB,CAAC,CAAS;IAC3C,OAAO,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAA;AACxC,CAAC;AAED,SAAS,kBAAkB,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAsB;IAC1F,OAAO,GAAG,QAAQ,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,mBAAmB,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,CAAA;AAClH,CAAC;AAED,SAAS,oBAAoB,CAAC,EAAE,IAAI,EAAE,EAAE,EAA2B;IACjE,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,qCAAqC,CAAC,CAAA;IACxF,CAAC;IAED,IAAI,CAAC,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,0BAA0B,EAAE,qCAAqC,CAAC,CAAA;IACpF,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC,CAAA;IACjC,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;IACrC,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,0BAA0B,CAAC,CAAA;IAEzG,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,uGAAuG,CAAC,CAAA;IAC1H,CAAC;IAED,OAAO;QACL,SAAS,EAAE,QAAQ;QACnB,gBAAgB;KACjB,CAAA;AACH,CAAC"}
1
+ {"version":3,"file":"downloaddatasets.js","sourceRoot":"","sources":["../src/downloaddatasets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAGzC,MAAM,iBAAiB,GAAG,EAAE,CAAA;AAC5B,MAAM,0BAA0B,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AACtD,MAAM,oBAAoB,GAAG,YAAY,CAAA;AAEzC,MAAM,OAAO,GAAG,UAAU,EAAE,CAAA;AAE5B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,uBAAgD;IACrF,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,uBAAuB,CAAA;IAC1E,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAA;IAC7G,MAAM,WAAW,GAAG,uBAAuB,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAA;IAClI,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAA;IACpG,MAAM,WAAW,GAAG,uBAAuB,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAA;IAChI,MAAM,YAAY,GAAG,uBAAuB,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB,CAAC,YAAY,CAAA;IAErH,0GAA0G;IAC1G,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;IAEjF,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;QACrC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAC,uBAAuB,CAAC,CAAA;YACrF,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAA;YAC3C,KAAK,CAAC,qDAAqD,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;YAElG,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;gBACzB,mHAAmH;gBACnH,MAAM,eAAe,CACnB,kBAAkB,CAAC;oBACjB,QAAQ;oBACR,MAAM;oBACN,MAAM;oBACN,WAAW;oBACX,QAAQ;oBACR,MAAM;oBACN,WAAW;oBACX,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,gBAAgB,GAAG,CAAC,CAAC;iBAC/C,CAAC,EACF,YAAY,CACb,CAAA;YACH,CAAC;YAED,iHAAiH;YACjH,MAAM,eAAe,CACnB,kBAAkB,CAAC;gBACjB,QAAQ;gBACR,MAAM;gBACN,MAAM;gBACN,WAAW;gBACX,QAAQ;gBACR,MAAM;gBACN,WAAW;gBACX,IAAI,EAAE,SAAS;aAChB,CAAC,EACF,YAAY,CACb,CAAA;YAED,6DAA6D;YAC7D,MAAM,iBAAiB,CACrB,QAAQ,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,gIAAgI;YACnK,iBAAiB,EACjB,CAAC,MAAM,EAAE,EAAE,CACT,eAAe,CACb,kBAAkB,CAAC;gBACjB,QAAQ;gBACR,MAAM;gBACN,MAAM;gBACN,WAAW;gBACX,QAAQ;gBACR,MAAM;gBACN,WAAW;gBACX,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;aACjC,CAAC,EACF,YAAY,CACb,CACJ,CAAA;YACD,MAAM,cAAc,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC,GAAG,IAAI,CAAA;YAErE,KAAK,CAAC,wEAAwE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,cAAc,CAAC,CAAA;QACvI,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,eAAgC,EAAE,YAAqB;IACpF,IAAI,YAAY,IAAI,UAAU,CAAC,eAAe,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7D,KAAK,CAAC,8CAA8C,EAAE,eAAe,CAAC,YAAY,CAAC,CAAA;QACnF,OAAM;IACR,CAAC;SAAM,CAAC;QACN,OAAO,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAA;IACxC,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,EAC1B,MAAM,EACN,QAAQ,EACR,QAAQ,EACR,IAAI,EACJ,MAAM,EACN,MAAM,EACN,WAAW,EACX,WAAW,EAUZ;IACC,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,EAAE,CAAA;IAClC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAA;IACjD,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAA;IAE1C,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,gBAAgB,IAAI,QAAQ,IAAI,QAAQ,IAAI,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,CAAA;IACpI,MAAM,QAAQ,GAAG,WAAW,CAAC;QAC3B,QAAQ;QACR,IAAI;QACJ,QAAQ;QACR,MAAM;QACN,MAAM;KACP,CAAC,CAAA;IAEF,MAAM,YAAY,GAAG,GAAG,WAAW,IAAI,QAAQ,EAAE,CAAA;IAEjD,OAAO;QACL,GAAG;QACH,YAAY;QACZ,SAAS,EAAE,OAAO,CAAC,UAAU;QAC7B,MAAM;KACP,CAAA;AACH,CAAC;AAID,MAAM,UAAU,mBAAmB,CAAC,CAAS;IAC3C,OAAO,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAA;AACxC,CAAC;AAED,SAAS,kBAAkB,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAsB;IAC1F,OAAO,GAAG,QAAQ,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,mBAAmB,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,CAAA;AAClH,CAAC;AAED,SAAS,oBAAoB,CAAC,EAAE,IAAI,EAAE,EAAE,EAA2B;IACjE,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,qCAAqC,CAAC,CAAA;IACxF,CAAC;IAED,IAAI,CAAC,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,0BAA0B,EAAE,qCAAqC,CAAC,CAAA;IACpF,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC,CAAA;IACjC,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;IACrC,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,0BAA0B,CAAC,CAAA;IAEzG,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,uGAAuG,CAAC,CAAA;IAC1H,CAAC;IAED,OAAO;QACL,SAAS,EAAE,QAAQ;QACnB,gBAAgB;KACjB,CAAA;AACH,CAAC"}
@@ -0,0 +1,2 @@
1
+ export declare function forEachConcurrent<T>(items: Iterable<T>, concurrency: number, action: (item: T) => Promise<unknown>): Promise<void>;
2
+ //# sourceMappingURL=foreachconcurrent.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"foreachconcurrent.d.ts","sourceRoot":"","sources":["../src/foreachconcurrent.ts"],"names":[],"mappings":"AAAA,wBAAsB,iBAAiB,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,iBA4BxH"}
@@ -0,0 +1,28 @@
1
+ export async function forEachConcurrent(items, concurrency, action) {
2
+ const iterator = items[Symbol.iterator]();
3
+ let hasError = false;
4
+ let firstError;
5
+ const runWorker = async () => {
6
+ while (hasError === false) {
7
+ const next = iterator.next();
8
+ if (next.done) {
9
+ return;
10
+ }
11
+ try {
12
+ await action(next.value);
13
+ }
14
+ catch (error) {
15
+ if (hasError === false) {
16
+ hasError = true;
17
+ firstError = error;
18
+ }
19
+ return;
20
+ }
21
+ }
22
+ };
23
+ await Promise.all(Array.from({ length: concurrency }, () => runWorker()));
24
+ if (hasError) {
25
+ throw firstError;
26
+ }
27
+ }
28
+ //# sourceMappingURL=foreachconcurrent.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"foreachconcurrent.js","sourceRoot":"","sources":["../src/foreachconcurrent.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAI,KAAkB,EAAE,WAAmB,EAAE,MAAqC;IACvH,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IACzC,IAAI,QAAQ,GAAG,KAAK,CAAA;IACpB,IAAI,UAAmB,CAAA;IAEvB,MAAM,SAAS,GAAG,KAAK,IAAI,EAAE;QAC3B,OAAO,QAAQ,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAA;YAC5B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,OAAM;YACR,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC1B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;oBACvB,QAAQ,GAAG,IAAI,CAAA;oBACf,UAAU,GAAG,KAAK,CAAA;gBACpB,CAAC;gBACD,OAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;IACzE,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,UAAU,CAAA;IAClB,CAAC;AACH,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAoM7C,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,mBAAmB,EAAE,mBAAmB,CAAA;IACxC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,IAAI,CAAA;IACd,MAAM,EAAE,IAAI,CAAA;IACZ,QAAQ,EAAE,QAAQ,CAAA;IAClB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IACtB,2BAA2B,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAC/C,CAAA;AAED,0BAAkB,YAAY;IAC5B,gBAAgB,qBAAqB;IACrC,kBAAkB,uBAAuB;CAC1C"}
1
+ {"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAgM7C,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,mBAAmB,EAAE,mBAAmB,CAAA;IACxC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,IAAI,CAAA;IACd,MAAM,EAAE,IAAI,CAAA;IACZ,QAAQ,EAAE,QAAQ,CAAA;IAClB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IACtB,2BAA2B,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAC/C,CAAA;AAED,0BAAkB,YAAY;IAC5B,gBAAgB,qBAAqB;IACrC,kBAAkB,uBAAuB;CAC1C"}
package/dist/worker.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import dbg from 'debug';
2
2
  import { existsSync } from 'node:fs';
3
- import pMap from 'p-map';
4
3
  import { isMainThread, parentPort, workerData } from 'worker_threads';
4
+ import { forEachConcurrent } from "./foreachconcurrent.js";
5
5
  import { addMinutes, download, formatDateToPath, optimizeFilters, sha256, wait, cleanTempFiles } from "./handy.js";
6
6
  const debug = dbg('tardis-dev');
7
7
  const DEFAULT_DATA_FEED_SLICE_SIZE = 1;
@@ -82,10 +82,10 @@ async function getAvailableDataFeedSlices(payload, filters, cacheDir, minutesCou
82
82
  sliceOffsets.push(offset);
83
83
  }
84
84
  // it both begining and end date of the range is accessible fetch all remaning slices concurently with CONCURRENCY_LIMIT
85
- await pMap(sliceOffsets, async (offset) => {
85
+ await forEachConcurrent(sliceOffsets, concurrencyLimit, async (offset) => {
86
86
  const requestedSliceSize = Math.min(replaySliceSize, minutesCountToFetch - 1 - offset);
87
87
  await getDataFeedSlice(payload, offset, filters, cacheDir, requestedSliceSize);
88
- }, { concurrency: concurrencyLimit });
88
+ });
89
89
  }
90
90
  async function getDataFeedSlice({ exchange, fromDate, endpoint, apiKey, dataFeedCompression, userAgent }, offset, filters, cacheDir, requestedSliceSize = DEFAULT_DATA_FEED_SLICE_SIZE, useCache = true) {
91
91
  const sliceTimestamp = addMinutes(fromDate, offset);
@@ -1 +1 @@
1
- {"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,OAAO,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,IAAI,MAAM,OAAO,CAAA;AACxB,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AACrE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,eAAe,EAAY,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAG5H,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,CAAA;AAE/B,MAAM,4BAA4B,GAAG,CAAC,CAAA;AAEtC,IAAI,YAAY,EAAE,CAAC;IACjB,KAAK,CAAC,mDAAmD,CAAC,CAAA;AAC5D,CAAC;KAAM,CAAC;IACN,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,MAAoB,EAAE,EAAE;QACjD,IAAI,MAAM,2DAAkC,EAAE,CAAC;YAC7C,cAAc,EAAE,CAAA;YAChB,UAAW,CAAC,WAAW,4DAAiC,CAAA;QAC1D,CAAC;IACH,CAAC,CAAC,CAAA;IACF,iBAAiB,CAAC,UAA8B,CAAC,CAAA;AACnD,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;IAChD,KAAK,CAAC,wCAAwC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;IAC7D,MAAM,GAAG,CAAA;AACX,CAAC,CAAC,CAAA;AAEF,KAAK,UAAU,iBAAiB,CAAC,OAAyB;IACxD,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,CAAA;IACxC,MAAM,uCAAuC,GAAG,CAAC,CAAA;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAA;IAC5B,mEAAmE;IACnE,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEhD,2HAA2H;IAC3H,qEAAqE;IACrE,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,sBAAsB,CAAC,CAAA;IAExH,gEAAgE;IAChE,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,QAAQ,UAAU,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;IAEnF,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,2BAA2B,KAAK,QAAQ;QACrD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,2BAA2B,EAAE,uCAAuC,CAAC;QACxF,CAAC,CAAC,EAAE,CAAA;IACR,MAAM,YAAY,GAAG,iBAAiB,GAAG,sBAAsB,CAAA;IAE/D,MAAM,0CAA0C,GAAG,MAAM,wBAAwB,CAC/E,OAAO,EACP,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,CACvB,CAAA;IAED,MAAM,0BAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,0CAA0C,EAAE,iBAAiB,CAAC,CAAA;IAE3H,2DAA2D;IAC3D,KAAK,IAAI,MAAM,GAAG,0CAA0C,EAAE,MAAM,GAAG,mBAAmB,EAAE,MAAM,EAAE,EAAE,CAAC;QACrG,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,GAAG,sBAAsB,CAAA;QACrF,MAAM,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;QAEzE,IAAI,gBAAgB,GAAG,6BAA6B,EAAE,CAAC;YACrD,MAAM,QAAQ,GAAG,gBAAgB,GAAG,6BAA6B,GAAG,GAAG,CAAA;YAEvE,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtB,CAAC;QACD,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAA;IAC5D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,OAAyB,EACzB,mBAA2B,EAC3B,YAAoB,EACpB,oBAA4B;IAE5B,MAAM,0BAA0B,GAAG,OAAO,CAAC,2BAA2B,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACxI,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAChC,OAAO,mBAAmB,CAAA;IAC5B,CAAC;IAED,IAAI,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IAEvE,4EAA4E;IAC5E,IAAI,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/D,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,6BAA6B,CAAA;QAClF,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,CAAA;QAC7B,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACnE,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,oBAAoB,CAAC,CAAA;IAC7H,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAA;AAC1E,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,OAAyB,EACzB,OAAiB,EACjB,QAAgB,EAChB,mBAA2B,EAC3B,gBAAwB;IAExB,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAM;IACR,CAAC;IAED,iHAAiH;IACjH,qEAAqE;IACrE,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAElI,kGAAkG;IAClG,MAAM,UAAU,GACd,mBAAmB,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAEpI,MAAM,eAAe,GACnB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB,CAAC,CAAA;IAC7H,MAAM,YAAY,GAAa,EAAE,CAAA;IACjC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,mBAAmB,GAAG,CAAC,EAAE,MAAM,IAAI,eAAe,EAAE,CAAC;QACjF,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC;IAED,wHAAwH;IACxH,MAAM,IAAI,CACR,YAAY,EACZ,KAAK,EAAE,MAAM,EAAE,EAAE;QACf,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,mBAAmB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAA;QACtF,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;IAChF,CAAC,EACD,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAClC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAoB,EAC1F,MAAc,EACd,OAAiB,EACjB,QAAgB,EAChB,kBAAkB,GAAG,4BAA4B,EACjD,QAAQ,GAAG,IAAI;IAEf,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,eAAe,GAAG,kBAAkB,KAAK,4BAA4B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,kBAAkB,EAAE,CAAA;IAChH,MAAM,aAAa,GAAG,GAAG,QAAQ,IAAI,gBAAgB,CAAC,cAAc,CAAC,GAAG,eAAe,OAAO,CAAA;IAC9F,MAAM,aAAa,GAAG,GAAG,aAAa,MAAM,CAAA;IAC5C,MAAM,aAAa,GAAG,GAAG,aAAa,KAAK,CAAA;IAC3C,IAAI,eAAe,CAAA;IACnB,IAAI,QAAQ,EAAE,CAAC;QACb,eAAe,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAA;IACrH,CAAC;IAED,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,KAAK,CAAC,oDAAoD,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;QACzF,MAAM,OAAO,GAAkB;YAC7B,QAAQ;YACR,SAAS,EAAE,eAAe;YAC1B,SAAS,EAAE,kBAAkB;SAC9B,CAAA;QACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QAChC,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,kBAAkB,EAAE,4BAA4B;SACjD,CAAA;IACH,CAAC;IAED,IAAI,GAAG,GAAG,GAAG,QAAQ,eAAe,QAAQ,SAAS,QAAQ,CAAC,WAAW,EAAE,WAAW,MAAM,gBAAgB,mBAAmB,EAAE,CAAA;IACjI,IAAI,kBAAkB,GAAG,4BAA4B,EAAE,CAAC;QACtD,GAAG,IAAI,cAAc,kBAAkB,EAAE,CAAA;IAC3C,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,IAAI,YAAY,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC;QACpC,MAAM;QACN,YAAY,EAAE,aAAa;QAC3B,GAAG;QACH,SAAS;QACT,8BAA8B,EAAE,IAAI;QACpC,cAAc,EAAE,mBAAmB,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY;KACvE,CAAC,CAAA;IACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAA;IACxE,MAAM,kBAAkB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,wBAAwB,CAAC,IAAI,4BAA4B,CAAC,CAAA;IAEnH,KAAK,CAAC,iEAAiE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAA;IACrG,MAAM,OAAO,GAAkB;QAC7B,QAAQ;QACR,SAAS,EAAE,cAAc,CAAC,YAAY;QACtC,SAAS,EAAE,iBAAiB;KAC7B,CAAA;IACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;IAEhC,OAAO;QACL,SAAS,EAAE,iBAAiB;QAC5B,kBAAkB;KACnB,CAAA;AACH,CAAC"}
1
+ {"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,OAAO,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAGlH,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,CAAA;AAE/B,MAAM,4BAA4B,GAAG,CAAC,CAAA;AAEtC,IAAI,YAAY,EAAE,CAAC;IACjB,KAAK,CAAC,mDAAmD,CAAC,CAAA;AAC5D,CAAC;KAAM,CAAC;IACN,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,MAAoB,EAAE,EAAE;QACjD,IAAI,MAAM,2DAAkC,EAAE,CAAC;YAC7C,cAAc,EAAE,CAAA;YAChB,UAAW,CAAC,WAAW,4DAAiC,CAAA;QAC1D,CAAC;IACH,CAAC,CAAC,CAAA;IACF,iBAAiB,CAAC,UAA8B,CAAC,CAAA;AACnD,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;IAChD,KAAK,CAAC,wCAAwC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;IAC7D,MAAM,GAAG,CAAA;AACX,CAAC,CAAC,CAAA;AAEF,KAAK,UAAU,iBAAiB,CAAC,OAAyB;IACxD,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,CAAA;IACxC,MAAM,uCAAuC,GAAG,CAAC,CAAA;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAA;IAC5B,mEAAmE;IACnE,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEhD,2HAA2H;IAC3H,qEAAqE;IACrE,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,sBAAsB,CAAC,CAAA;IAExH,gEAAgE;IAChE,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,QAAQ,UAAU,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;IAEnF,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,2BAA2B,KAAK,QAAQ;QACrD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,2BAA2B,EAAE,uCAAuC,CAAC;QACxF,CAAC,CAAC,EAAE,CAAA;IACR,MAAM,YAAY,GAAG,iBAAiB,GAAG,sBAAsB,CAAA;IAE/D,MAAM,0CAA0C,GAAG,MAAM,wBAAwB,CAC/E,OAAO,EACP,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,CACvB,CAAA;IAED,MAAM,0BAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,0CAA0C,EAAE,iBAAiB,CAAC,CAAA;IAE3H,2DAA2D;IAC3D,KAAK,IAAI,MAAM,GAAG,0CAA0C,EAAE,MAAM,GAAG,mBAAmB,EAAE,MAAM,EAAE,EAAE,CAAC;QACrG,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,GAAG,sBAAsB,CAAA;QACrF,MAAM,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;QAEzE,IAAI,gBAAgB,GAAG,6BAA6B,EAAE,CAAC;YACrD,MAAM,QAAQ,GAAG,gBAAgB,GAAG,6BAA6B,GAAG,GAAG,CAAA;YAEvE,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtB,CAAC;QACD,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAA;IAC5D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,OAAyB,EACzB,mBAA2B,EAC3B,YAAoB,EACpB,oBAA4B;IAE5B,MAAM,0BAA0B,GAAG,OAAO,CAAC,2BAA2B,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACxI,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAChC,OAAO,mBAAmB,CAAA;IAC5B,CAAC;IAED,IAAI,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IAEvE,4EAA4E;IAC5E,IAAI,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/D,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,6BAA6B,CAAA;QAClF,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,CAAA;QAC7B,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACnE,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,oBAAoB,CAAC,CAAA;IAC7H,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAA;AAC1E,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,OAAyB,EACzB,OAAiB,EACjB,QAAgB,EAChB,mBAA2B,EAC3B,gBAAwB;IAExB,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAM;IACR,CAAC;IAED,iHAAiH;IACjH,qEAAqE;IACrE,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAElI,kGAAkG;IAClG,MAAM,UAAU,GACd,mBAAmB,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAEpI,MAAM,eAAe,GACnB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB,CAAC,CAAA;IAC7H,MAAM,YAAY,GAAa,EAAE,CAAA;IACjC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,mBAAmB,GAAG,CAAC,EAAE,MAAM,IAAI,eAAe,EAAE,CAAC;QACjF,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC;IAED,wHAAwH;IACxH,MAAM,iBAAiB,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACvE,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,mBAAmB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAA;QACtF,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;IAChF,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAoB,EAC1F,MAAc,EACd,OAAiB,EACjB,QAAgB,EAChB,kBAAkB,GAAG,4BAA4B,EACjD,QAAQ,GAAG,IAAI;IAEf,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,eAAe,GAAG,kBAAkB,KAAK,4BAA4B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,kBAAkB,EAAE,CAAA;IAChH,MAAM,aAAa,GAAG,GAAG,QAAQ,IAAI,gBAAgB,CAAC,cAAc,CAAC,GAAG,eAAe,OAAO,CAAA;IAC9F,MAAM,aAAa,GAAG,GAAG,aAAa,MAAM,CAAA;IAC5C,MAAM,aAAa,GAAG,GAAG,aAAa,KAAK,CAAA;IAC3C,IAAI,eAAe,CAAA;IACnB,IAAI,QAAQ,EAAE,CAAC;QACb,eAAe,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAA;IACrH,CAAC;IAED,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,KAAK,CAAC,oDAAoD,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;QACzF,MAAM,OAAO,GAAkB;YAC7B,QAAQ;YACR,SAAS,EAAE,eAAe;YAC1B,SAAS,EAAE,kBAAkB;SAC9B,CAAA;QACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QAChC,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,kBAAkB,EAAE,4BAA4B;SACjD,CAAA;IACH,CAAC;IAED,IAAI,GAAG,GAAG,GAAG,QAAQ,eAAe,QAAQ,SAAS,QAAQ,CAAC,WAAW,EAAE,WAAW,MAAM,gBAAgB,mBAAmB,EAAE,CAAA;IACjI,IAAI,kBAAkB,GAAG,4BAA4B,EAAE,CAAC;QACtD,GAAG,IAAI,cAAc,kBAAkB,EAAE,CAAA;IAC3C,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,IAAI,YAAY,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC;QACpC,MAAM;QACN,YAAY,EAAE,aAAa;QAC3B,GAAG;QACH,SAAS;QACT,8BAA8B,EAAE,IAAI;QACpC,cAAc,EAAE,mBAAmB,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY;KACvE,CAAC,CAAA;IACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAA;IACxE,MAAM,kBAAkB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,wBAAwB,CAAC,IAAI,4BAA4B,CAAC,CAAA;IAEnH,KAAK,CAAC,iEAAiE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAA;IACrG,MAAM,OAAO,GAAkB;QAC7B,QAAQ;QACR,SAAS,EAAE,cAAc,CAAC,YAAY;QACtC,SAAS,EAAE,iBAAiB;KAC7B,CAAA;IACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;IAEhC,OAAO;QACL,SAAS,EAAE,iBAAiB;QAC5B,kBAAkB;KACnB,CAAA;AACH,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tardis-dev",
3
- "version": "18.0.0",
3
+ "version": "18.0.1",
4
4
  "engines": {
5
5
  "node": ">=24.5"
6
6
  },
@@ -70,7 +70,6 @@
70
70
  "dependencies": {
71
71
  "debug": "^4.4.3",
72
72
  "follow-redirects": "^1.16.0",
73
- "p-map": "^7.0.5",
74
73
  "protobufjs": "^8.7.1",
75
74
  "ws": "^8.21.1"
76
75
  },
package/src/combine.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { isManagedRealTimeIterator, mergeRealTime } from './realtimeiterator.ts'
1
+ import { closeIterator, isManagedRealTimeIterator, mergeRealTime } from './realtimeiterator.ts'
2
2
 
3
3
  type NextMessageResultWithIndex = {
4
4
  index: number
@@ -129,8 +129,6 @@ async function* combineHistorical(
129
129
  }
130
130
  } while (aliveIteratorsCount > 0)
131
131
  } finally {
132
- for (let iterator of iterators) {
133
- ;(iterator as any).return()
134
- }
132
+ await Promise.all(iterators.map((iterator) => closeIterator('stream' in iterator ? iterator.stream : iterator)))
135
133
  }
136
134
  }
@@ -1,7 +1,7 @@
1
1
  import { existsSync } from 'node:fs'
2
- import pMap from 'p-map'
3
2
  import { debug } from './debug.ts'
4
3
  import { DatasetType } from './exchangedetails.ts'
4
+ import { forEachConcurrent } from './foreachconcurrent.ts'
5
5
  import { addDays, doubleDigit, download, parseAsUTCDate, sequence } from './handy.ts'
6
6
  import { getOptions } from './options.ts'
7
7
  import { Exchange } from './types.ts'
@@ -62,8 +62,9 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
62
62
  )
63
63
 
64
64
  // download the rest concurrently up to the CONCURRENCY_LIMIT
65
- await pMap(
65
+ await forEachConcurrent(
66
66
  sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
67
+ CONCURRENCY_LIMIT,
67
68
  (offset) =>
68
69
  downloadDataSet(
69
70
  getDownloadOptions({
@@ -77,8 +78,7 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
77
78
  date: addDays(startDate, offset)
78
79
  }),
79
80
  skipIfExists
80
- ),
81
- { concurrency: CONCURRENCY_LIMIT }
81
+ )
82
82
  )
83
83
  const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000
84
84
 
@@ -0,0 +1,29 @@
1
+ export async function forEachConcurrent<T>(items: Iterable<T>, concurrency: number, action: (item: T) => Promise<unknown>) {
2
+ const iterator = items[Symbol.iterator]()
3
+ let hasError = false
4
+ let firstError: unknown
5
+
6
+ const runWorker = async () => {
7
+ while (hasError === false) {
8
+ const next = iterator.next()
9
+ if (next.done) {
10
+ return
11
+ }
12
+
13
+ try {
14
+ await action(next.value)
15
+ } catch (error) {
16
+ if (hasError === false) {
17
+ hasError = true
18
+ firstError = error
19
+ }
20
+ return
21
+ }
22
+ }
23
+ }
24
+
25
+ await Promise.all(Array.from({ length: concurrency }, () => runWorker()))
26
+ if (hasError) {
27
+ throw firstError
28
+ }
29
+ }
package/src/worker.ts CHANGED
@@ -1,8 +1,8 @@
1
1
  import dbg from 'debug'
2
2
  import { existsSync } from 'node:fs'
3
- import pMap from 'p-map'
4
3
  import { isMainThread, parentPort, workerData } from 'worker_threads'
5
- import { addMinutes, download, formatDateToPath, optimizeFilters, sequence, sha256, wait, cleanTempFiles } from './handy.ts'
4
+ import { forEachConcurrent } from './foreachconcurrent.ts'
5
+ import { addMinutes, download, formatDateToPath, optimizeFilters, sha256, wait, cleanTempFiles } from './handy.ts'
6
6
  import type { DataFeedCompression } from './options.ts'
7
7
  import { Exchange, Filter } from './types.ts'
8
8
  const debug = dbg('tardis-dev')
@@ -123,14 +123,10 @@ async function getAvailableDataFeedSlices(
123
123
  }
124
124
 
125
125
  // it both begining and end date of the range is accessible fetch all remaning slices concurently with CONCURRENCY_LIMIT
126
- await pMap(
127
- sliceOffsets,
128
- async (offset) => {
129
- const requestedSliceSize = Math.min(replaySliceSize, minutesCountToFetch - 1 - offset)
130
- await getDataFeedSlice(payload, offset, filters, cacheDir, requestedSliceSize)
131
- },
132
- { concurrency: concurrencyLimit }
133
- )
126
+ await forEachConcurrent(sliceOffsets, concurrencyLimit, async (offset) => {
127
+ const requestedSliceSize = Math.min(replaySliceSize, minutesCountToFetch - 1 - offset)
128
+ await getDataFeedSlice(payload, offset, filters, cacheDir, requestedSliceSize)
129
+ })
134
130
  }
135
131
 
136
132
  async function getDataFeedSlice(