tardis-dev 18.0.0 → 18.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/combine.js +2 -5
- package/dist/combine.js.map +1 -1
- package/dist/downloaddatasets.d.ts.map +1 -1
- package/dist/downloaddatasets.js +4 -4
- package/dist/downloaddatasets.js.map +1 -1
- package/dist/foreachconcurrent.d.ts +2 -0
- package/dist/foreachconcurrent.d.ts.map +1 -0
- package/dist/foreachconcurrent.js +28 -0
- package/dist/foreachconcurrent.js.map +1 -0
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +3 -3
- package/dist/worker.js.map +1 -1
- package/package.json +1 -2
- package/src/combine.ts +2 -4
- package/src/downloaddatasets.ts +4 -4
- package/src/foreachconcurrent.ts +29 -0
- package/src/worker.ts +6 -10
package/dist/combine.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isManagedRealTimeIterator, mergeRealTime } from "./realtimeiterator.js";
|
|
1
|
+
import { closeIterator, isManagedRealTimeIterator, mergeRealTime } from "./realtimeiterator.js";
|
|
2
2
|
const DATE_MAX = new Date(8640000000000000);
|
|
3
3
|
async function nextWithIndex(iterator, index) {
|
|
4
4
|
if ('offsetMS' in iterator) {
|
|
@@ -91,10 +91,7 @@ async function* combineHistorical(iterators) {
|
|
|
91
91
|
} while (aliveIteratorsCount > 0);
|
|
92
92
|
}
|
|
93
93
|
finally {
|
|
94
|
-
|
|
95
|
-
;
|
|
96
|
-
iterator.return();
|
|
97
|
-
}
|
|
94
|
+
await Promise.all(iterators.map((iterator) => closeIterator('stream' in iterator ? iterator.stream : iterator)));
|
|
98
95
|
}
|
|
99
96
|
}
|
|
100
97
|
//# sourceMappingURL=combine.js.map
|
package/dist/combine.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"combine.js","sourceRoot":"","sources":["../src/combine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;
|
|
1
|
+
{"version":3,"file":"combine.js","sourceRoot":"","sources":["../src/combine.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAS/F,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,gBAAgB,CAAC,CAAA;AAU3C,KAAK,UAAU,aAAa,CAC1B,QAA+G,EAC/G,KAAa;IAEb,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;QAE3C,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,OAAO,QAAQ,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAA;YAE9G,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACnB,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,kBAAkB,EAAE,GAAG,QAAQ,CAAC,CAAA;YAC7G,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM;YACN,KAAK;SACN,CAAA;IACH,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;QAEpC,OAAO;YACL,MAAM;YACN,KAAK;SACN,CAAA;IACH,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAkC,EAAE,OAAmC;IAC/F,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACvB,OAAO,MAAM,CAAA;IACf,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACxB,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,CAAA;IACtE,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,EAAE,CAAA;IAEpE,IAAI,gBAAgB,GAAG,eAAe,EAAE,CAAC;QACvC,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,IAAI,gBAAgB,KAAK,eAAe,EAAE,CAAC;QACzC,MAAM,4BAA4B,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC,CAAA;QAChF,MAAM,2BAA2B,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,IAAI,CAAC,CAAA;QAE9E,IAAI,4BAA4B,GAAG,2BAA2B,EAAE,CAAC;YAC/D,OAAO,OAAO,CAAA;QAChB,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,kEAAkE;AAClE,+CAA+C;AAC/C,MAAM,UAAU,OAAO,CAA4B,GAAG,gBAAmB;IACvE,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QACjD,IAAI,QAAQ,IAAI,OAAO,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC,MAAM,CAAA;QACvB,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC,CAAC,CAAA;IACF,IAAI,yBAAyB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,OAAO,aAAa,CAAC,SAAS,CAA8C,CAAA;IAC9E,CAAC;IAED,OAAO,iBAAiB,CAAC,gBAAgB,CAA8C,CAAA;AACzF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,iBAAiB,CAC/B,SAAoH;IAEpH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAM;IACR,CAAC;IAED,IAAI,CAAC;QACH,kCAAkC;QAClC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;QAC/D,IAAI,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAA;QACxC,GAAG,CAAC;YACF,+CAA+C;YAC/C,6FAA6F;YAE7F,yCAAyC;YACzC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;YACjE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,YAAY,CAAA;YAEtC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,mBAAmB,EAAE,CAAA;gBAErB,yEAAyE;gBACzE,0EAA0E;gBAC1E,uBAAuB;gBAEvB,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG;oBACtB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE;wBACL,cAAc,EAAE,QAAQ;qBACzB;iBACF,CAAA;YACH,CAAC;iBAAM,CAAC;gBACN,+EAA+E;gBAC/E,MAAM,MAAM,CAAC,KAAK,CAAA;gBAClB,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;YAC/D,CAAC;QACH,CAAC,QAAQ,mBAAmB,GAAG,CAAC,EAAC;IACnC,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;IAClH,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"downloaddatasets.d.ts","sourceRoot":"","sources":["../src/downloaddatasets.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"downloaddatasets.d.ts","sourceRoot":"","sources":["../src/downloaddatasets.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAIlD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAQrC,wBAAsB,gBAAgB,CAAC,uBAAuB,EAAE,uBAAuB,iBAyEtF;AAuDD,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,UAE5C;AA6BD,KAAK,kBAAkB,GAAG;IACxB,QAAQ,EAAE,QAAQ,CAAA;IAClB,QAAQ,EAAE,WAAW,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,IAAI,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;CACf,CAAA;AAED,KAAK,uBAAuB,GAAG;IAC7B,QAAQ,EAAE,QAAQ,CAAA;IAClB,SAAS,EAAE,WAAW,EAAE,CAAA;IACxB,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,CAAC,EAAE,KAAK,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,CAAC,OAAO,EAAE,kBAAkB,KAAK,MAAM,CAAA;IACrD,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB,CAAA"}
|
package/dist/downloaddatasets.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
|
-
import pMap from 'p-map';
|
|
3
2
|
import { debug } from "./debug.js";
|
|
3
|
+
import { forEachConcurrent } from "./foreachconcurrent.js";
|
|
4
4
|
import { addDays, doubleDigit, download, parseAsUTCDate, sequence } from "./handy.js";
|
|
5
5
|
import { getOptions } from "./options.js";
|
|
6
6
|
const CONCURRENCY_LIMIT = 20;
|
|
@@ -46,8 +46,8 @@ export async function downloadDatasets(downloadDatasetsOptions) {
|
|
|
46
46
|
date: startDate
|
|
47
47
|
}), skipIfExists);
|
|
48
48
|
// download the rest concurrently up to the CONCURRENCY_LIMIT
|
|
49
|
-
await
|
|
50
|
-
(offset) => downloadDataSet(getDownloadOptions({
|
|
49
|
+
await forEachConcurrent(sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
|
|
50
|
+
CONCURRENCY_LIMIT, (offset) => downloadDataSet(getDownloadOptions({
|
|
51
51
|
exchange,
|
|
52
52
|
symbol,
|
|
53
53
|
apiKey,
|
|
@@ -56,7 +56,7 @@ export async function downloadDatasets(downloadDatasetsOptions) {
|
|
|
56
56
|
format,
|
|
57
57
|
getFilename,
|
|
58
58
|
date: addDays(startDate, offset)
|
|
59
|
-
}), skipIfExists)
|
|
59
|
+
}), skipIfExists));
|
|
60
60
|
const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000;
|
|
61
61
|
debug('dataset download finished for %s %s %s from %s to %s, time: %s seconds', exchange, dataType, symbol, from, to, elapsedSeconds);
|
|
62
62
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"downloaddatasets.js","sourceRoot":"","sources":["../src/downloaddatasets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,
|
|
1
|
+
{"version":3,"file":"downloaddatasets.js","sourceRoot":"","sources":["../src/downloaddatasets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAGzC,MAAM,iBAAiB,GAAG,EAAE,CAAA;AAC5B,MAAM,0BAA0B,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AACtD,MAAM,oBAAoB,GAAG,YAAY,CAAA;AAEzC,MAAM,OAAO,GAAG,UAAU,EAAE,CAAA;AAE5B,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,uBAAgD;IACrF,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,uBAAuB,CAAA;IAC1E,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAA;IAC7G,MAAM,WAAW,GAAG,uBAAuB,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC,CAAC,oBAAoB,CAAA;IAClI,MAAM,MAAM,GAAG,uBAAuB,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAA;IACpG,MAAM,WAAW,GAAG,uBAAuB,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,WAAW,CAAC,CAAC,CAAC,kBAAkB,CAAA;IAChI,MAAM,YAAY,GAAG,uBAAuB,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,uBAAuB,CAAC,YAAY,CAAA;IAErH,0GAA0G;IAC1G,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;IAEjF,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;QACrC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,oBAAoB,CAAC,uBAAuB,CAAC,CAAA;YACrF,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAA;YAC3C,KAAK,CAAC,qDAAqD,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;YAElG,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;gBACzB,mHAAmH;gBACnH,MAAM,eAAe,CACnB,kBAAkB,CAAC;oBACjB,QAAQ;oBACR,MAAM;oBACN,MAAM;oBACN,WAAW;oBACX,QAAQ;oBACR,MAAM;oBACN,WAAW;oBACX,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,gBAAgB,GAAG,CAAC,CAAC;iBAC/C,CAAC,EACF,YAAY,CACb,CAAA;YACH,CAAC;YAED,iHAAiH;YACjH,MAAM,eAAe,CACnB,kBAAkB,CAAC;gBACjB,QAAQ;gBACR,MAAM;gBACN,MAAM;gBACN,WAAW;gBACX,QAAQ;gBACR,MAAM;gBACN,WAAW;gBACX,IAAI,EAAE,SAAS;aAChB,CAAC,EACF,YAAY,CACb,CAAA;YAED,6DAA6D;YAC7D,MAAM,iBAAiB,CACrB,QAAQ,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,gIAAgI;YACnK,iBAAiB,EACjB,CAAC,MAAM,EAAE,EAAE,CACT,eAAe,CACb,kBAAkB,CAAC;gBACjB,QAAQ;gBACR,MAAM;gBACN,MAAM;gBACN,WAAW;gBACX,QAAQ;gBACR,MAAM;gBACN,WAAW;gBACX,IAAI,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;aACjC,CAAC,EACF,YAAY,CACb,CACJ,CAAA;YACD,MAAM,cAAc,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,cAAc,CAAC,GAAG,IAAI,CAAA;YAErE,KAAK,CAAC,wEAAwE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,cAAc,CAAC,CAAA;QACvI,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,eAAgC,EAAE,YAAqB;IACpF,IAAI,YAAY,IAAI,UAAU,CAAC,eAAe,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7D,KAAK,CAAC,8CAA8C,EAAE,eAAe,CAAC,YAAY,CAAC,CAAA;QACnF,OAAM;IACR,CAAC;SAAM,CAAC;QACN,OAAO,MAAM,QAAQ,CAAC,eAAe,CAAC,CAAA;IACxC,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,EAC1B,MAAM,EACN,QAAQ,EACR,QAAQ,EACR,IAAI,EACJ,MAAM,EACN,MAAM,EACN,WAAW,EACX,WAAW,EAUZ;IACC,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,EAAE,CAAA;IAClC,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAA;IACjD,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAA;IAE1C,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,gBAAgB,IAAI,QAAQ,IAAI,QAAQ,IAAI,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,CAAA;IACpI,MAAM,QAAQ,GAAG,WAAW,CAAC;QAC3B,QAAQ;QACR,IAAI;QACJ,QAAQ;QACR,MAAM;QACN,MAAM;KACP,CAAC,CAAA;IAEF,MAAM,YAAY,GAAG,GAAG,WAAW,IAAI,QAAQ,EAAE,CAAA;IAEjD,OAAO;QACL,GAAG;QACH,YAAY;QACZ,SAAS,EAAE,OAAO,CAAC,UAAU;QAC7B,MAAM;KACP,CAAA;AACH,CAAC;AAID,MAAM,UAAU,mBAAmB,CAAC,CAAS;IAC3C,OAAO,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAA;AACxC,CAAC;AAED,SAAS,kBAAkB,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAsB;IAC1F,OAAO,GAAG,QAAQ,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,mBAAmB,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,CAAA;AAClH,CAAC;AAED,SAAS,oBAAoB,CAAC,EAAE,IAAI,EAAE,EAAE,EAA2B;IACjE,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,4BAA4B,IAAI,qCAAqC,CAAC,CAAA;IACxF,CAAC;IAED,IAAI,CAAC,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,0BAA0B,EAAE,qCAAqC,CAAC,CAAA;IACpF,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,EAAE,CAAC,CAAA;IACjC,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;IACrC,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,0BAA0B,CAAC,CAAA;IAEzG,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,uGAAuG,CAAC,CAAA;IAC1H,CAAC;IAED,OAAO;QACL,SAAS,EAAE,QAAQ;QACnB,gBAAgB;KACjB,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"foreachconcurrent.d.ts","sourceRoot":"","sources":["../src/foreachconcurrent.ts"],"names":[],"mappings":"AAAA,wBAAsB,iBAAiB,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,iBA4BxH"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export async function forEachConcurrent(items, concurrency, action) {
|
|
2
|
+
const iterator = items[Symbol.iterator]();
|
|
3
|
+
let hasError = false;
|
|
4
|
+
let firstError;
|
|
5
|
+
const runWorker = async () => {
|
|
6
|
+
while (hasError === false) {
|
|
7
|
+
const next = iterator.next();
|
|
8
|
+
if (next.done) {
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
try {
|
|
12
|
+
await action(next.value);
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
if (hasError === false) {
|
|
16
|
+
hasError = true;
|
|
17
|
+
firstError = error;
|
|
18
|
+
}
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
await Promise.all(Array.from({ length: concurrency }, () => runWorker()));
|
|
24
|
+
if (hasError) {
|
|
25
|
+
throw firstError;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=foreachconcurrent.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"foreachconcurrent.js","sourceRoot":"","sources":["../src/foreachconcurrent.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAI,KAAkB,EAAE,WAAmB,EAAE,MAAqC;IACvH,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IACzC,IAAI,QAAQ,GAAG,KAAK,CAAA;IACpB,IAAI,UAAmB,CAAA;IAEvB,MAAM,SAAS,GAAG,KAAK,IAAI,EAAE;QAC3B,OAAO,QAAQ,KAAK,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAA;YAC5B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,OAAM;YACR,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC1B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;oBACvB,QAAQ,GAAG,IAAI,CAAA;oBACf,UAAU,GAAG,KAAK,CAAA;gBACpB,CAAC;gBACD,OAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;IACzE,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,UAAU,CAAA;IAClB,CAAC;AACH,CAAC"}
|
package/dist/worker.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAgM7C,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,mBAAmB,EAAE,mBAAmB,CAAA;IACxC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,IAAI,CAAA;IACd,MAAM,EAAE,IAAI,CAAA;IACZ,QAAQ,EAAE,QAAQ,CAAA;IAClB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IACtB,2BAA2B,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAC/C,CAAA;AAED,0BAAkB,YAAY;IAC5B,gBAAgB,qBAAqB;IACrC,kBAAkB,uBAAuB;CAC1C"}
|
package/dist/worker.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import dbg from 'debug';
|
|
2
2
|
import { existsSync } from 'node:fs';
|
|
3
|
-
import pMap from 'p-map';
|
|
4
3
|
import { isMainThread, parentPort, workerData } from 'worker_threads';
|
|
4
|
+
import { forEachConcurrent } from "./foreachconcurrent.js";
|
|
5
5
|
import { addMinutes, download, formatDateToPath, optimizeFilters, sha256, wait, cleanTempFiles } from "./handy.js";
|
|
6
6
|
const debug = dbg('tardis-dev');
|
|
7
7
|
const DEFAULT_DATA_FEED_SLICE_SIZE = 1;
|
|
@@ -82,10 +82,10 @@ async function getAvailableDataFeedSlices(payload, filters, cacheDir, minutesCou
|
|
|
82
82
|
sliceOffsets.push(offset);
|
|
83
83
|
}
|
|
84
84
|
// it both begining and end date of the range is accessible fetch all remaning slices concurently with CONCURRENCY_LIMIT
|
|
85
|
-
await
|
|
85
|
+
await forEachConcurrent(sliceOffsets, concurrencyLimit, async (offset) => {
|
|
86
86
|
const requestedSliceSize = Math.min(replaySliceSize, minutesCountToFetch - 1 - offset);
|
|
87
87
|
await getDataFeedSlice(payload, offset, filters, cacheDir, requestedSliceSize);
|
|
88
|
-
}
|
|
88
|
+
});
|
|
89
89
|
}
|
|
90
90
|
async function getDataFeedSlice({ exchange, fromDate, endpoint, apiKey, dataFeedCompression, userAgent }, offset, filters, cacheDir, requestedSliceSize = DEFAULT_DATA_FEED_SLICE_SIZE, useCache = true) {
|
|
91
91
|
const sliceTimestamp = addMinutes(fromDate, offset);
|
package/dist/worker.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,OAAO,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,OAAO,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAGlH,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,CAAA;AAE/B,MAAM,4BAA4B,GAAG,CAAC,CAAA;AAEtC,IAAI,YAAY,EAAE,CAAC;IACjB,KAAK,CAAC,mDAAmD,CAAC,CAAA;AAC5D,CAAC;KAAM,CAAC;IACN,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,MAAoB,EAAE,EAAE;QACjD,IAAI,MAAM,2DAAkC,EAAE,CAAC;YAC7C,cAAc,EAAE,CAAA;YAChB,UAAW,CAAC,WAAW,4DAAiC,CAAA;QAC1D,CAAC;IACH,CAAC,CAAC,CAAA;IACF,iBAAiB,CAAC,UAA8B,CAAC,CAAA;AACnD,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;IAChD,KAAK,CAAC,wCAAwC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;IAC7D,MAAM,GAAG,CAAA;AACX,CAAC,CAAC,CAAA;AAEF,KAAK,UAAU,iBAAiB,CAAC,OAAyB;IACxD,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,CAAA;IACxC,MAAM,uCAAuC,GAAG,CAAC,CAAA;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAA;IAC5B,mEAAmE;IACnE,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEhD,2HAA2H;IAC3H,qEAAqE;IACrE,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,sBAAsB,CAAC,CAAA;IAExH,gEAAgE;IAChE,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,QAAQ,UAAU,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;IAEnF,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,2BAA2B,KAAK,QAAQ;QACrD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,2BAA2B,EAAE,uCAAuC,CAAC;QACxF,CAAC,CAAC,EAAE,CAAA;IACR,MAAM,YAAY,GAAG,iBAAiB,GAAG,sBAAsB,CAAA;IAE/D,MAAM,0CAA0C,GAAG,MAAM,wBAAwB,CAC/E,OAAO,EACP,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,CACvB,CAAA;IAED,MAAM,0BAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,0CAA0C,EAAE,iBAAiB,CAAC,CAAA;IAE3H,2DAA2D;IAC3D,KAAK,IAAI,MAAM,GAAG,0CAA0C,EAAE,MAAM,GAAG,mBAAmB,EAAE,MAAM,EAAE,EAAE,CAAC;QACrG,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,GAAG,sBAAsB,CAAA;QACrF,MAAM,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;QAEzE,IAAI,gBAAgB,GAAG,6BAA6B,EAAE,CAAC;YACrD,MAAM,QAAQ,GAAG,gBAAgB,GAAG,6BAA6B,GAAG,GAAG,CAAA;YAEvE,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtB,CAAC;QACD,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAA;IAC5D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,OAAyB,EACzB,mBAA2B,EAC3B,YAAoB,EACpB,oBAA4B;IAE5B,MAAM,0BAA0B,GAAG,OAAO,CAAC,2BAA2B,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACxI,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAChC,OAAO,mBAAmB,CAAA;IAC5B,CAAC;IAED,IAAI,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IAEvE,4EAA4E;IAC5E,IAAI,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/D,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,6BAA6B,CAAA;QAClF,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,CAAA;QAC7B,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACnE,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,oBAAoB,CAAC,CAAA;IAC7H,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAA;AAC1E,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,OAAyB,EACzB,OAAiB,EACjB,QAAgB,EAChB,mBAA2B,EAC3B,gBAAwB;IAExB,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAM;IACR,CAAC;IAED,iHAAiH;IACjH,qEAAqE;IACrE,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAElI,kGAAkG;IAClG,MAAM,UAAU,GACd,mBAAmB,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAEpI,MAAM,eAAe,GACnB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB,CAAC,CAAA;IAC7H,MAAM,YAAY,GAAa,EAAE,CAAA;IACjC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,mBAAmB,GAAG,CAAC,EAAE,MAAM,IAAI,eAAe,EAAE,CAAC;QACjF,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC;IAED,wHAAwH;IACxH,MAAM,iBAAiB,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACvE,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,mBAAmB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAA;QACtF,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;IAChF,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAoB,EAC1F,MAAc,EACd,OAAiB,EACjB,QAAgB,EAChB,kBAAkB,GAAG,4BAA4B,EACjD,QAAQ,GAAG,IAAI;IAEf,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,eAAe,GAAG,kBAAkB,KAAK,4BAA4B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,kBAAkB,EAAE,CAAA;IAChH,MAAM,aAAa,GAAG,GAAG,QAAQ,IAAI,gBAAgB,CAAC,cAAc,CAAC,GAAG,eAAe,OAAO,CAAA;IAC9F,MAAM,aAAa,GAAG,GAAG,aAAa,MAAM,CAAA;IAC5C,MAAM,aAAa,GAAG,GAAG,aAAa,KAAK,CAAA;IAC3C,IAAI,eAAe,CAAA;IACnB,IAAI,QAAQ,EAAE,CAAC;QACb,eAAe,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAA;IACrH,CAAC;IAED,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,KAAK,CAAC,oDAAoD,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;QACzF,MAAM,OAAO,GAAkB;YAC7B,QAAQ;YACR,SAAS,EAAE,eAAe;YAC1B,SAAS,EAAE,kBAAkB;SAC9B,CAAA;QACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QAChC,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,kBAAkB,EAAE,4BAA4B;SACjD,CAAA;IACH,CAAC;IAED,IAAI,GAAG,GAAG,GAAG,QAAQ,eAAe,QAAQ,SAAS,QAAQ,CAAC,WAAW,EAAE,WAAW,MAAM,gBAAgB,mBAAmB,EAAE,CAAA;IACjI,IAAI,kBAAkB,GAAG,4BAA4B,EAAE,CAAC;QACtD,GAAG,IAAI,cAAc,kBAAkB,EAAE,CAAA;IAC3C,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,IAAI,YAAY,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC;QACpC,MAAM;QACN,YAAY,EAAE,aAAa;QAC3B,GAAG;QACH,SAAS;QACT,8BAA8B,EAAE,IAAI;QACpC,cAAc,EAAE,mBAAmB,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY;KACvE,CAAC,CAAA;IACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAA;IACxE,MAAM,kBAAkB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,wBAAwB,CAAC,IAAI,4BAA4B,CAAC,CAAA;IAEnH,KAAK,CAAC,iEAAiE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAA;IACrG,MAAM,OAAO,GAAkB;QAC7B,QAAQ;QACR,SAAS,EAAE,cAAc,CAAC,YAAY;QACtC,SAAS,EAAE,iBAAiB;KAC7B,CAAA;IACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;IAEhC,OAAO;QACL,SAAS,EAAE,iBAAiB;QAC5B,kBAAkB;KACnB,CAAA;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tardis-dev",
|
|
3
|
-
"version": "18.0.
|
|
3
|
+
"version": "18.0.1",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=24.5"
|
|
6
6
|
},
|
|
@@ -70,7 +70,6 @@
|
|
|
70
70
|
"dependencies": {
|
|
71
71
|
"debug": "^4.4.3",
|
|
72
72
|
"follow-redirects": "^1.16.0",
|
|
73
|
-
"p-map": "^7.0.5",
|
|
74
73
|
"protobufjs": "^8.7.1",
|
|
75
74
|
"ws": "^8.21.1"
|
|
76
75
|
},
|
package/src/combine.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isManagedRealTimeIterator, mergeRealTime } from './realtimeiterator.ts'
|
|
1
|
+
import { closeIterator, isManagedRealTimeIterator, mergeRealTime } from './realtimeiterator.ts'
|
|
2
2
|
|
|
3
3
|
type NextMessageResultWithIndex = {
|
|
4
4
|
index: number
|
|
@@ -129,8 +129,6 @@ async function* combineHistorical(
|
|
|
129
129
|
}
|
|
130
130
|
} while (aliveIteratorsCount > 0)
|
|
131
131
|
} finally {
|
|
132
|
-
|
|
133
|
-
;(iterator as any).return()
|
|
134
|
-
}
|
|
132
|
+
await Promise.all(iterators.map((iterator) => closeIterator('stream' in iterator ? iterator.stream : iterator)))
|
|
135
133
|
}
|
|
136
134
|
}
|
package/src/downloaddatasets.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs'
|
|
2
|
-
import pMap from 'p-map'
|
|
3
2
|
import { debug } from './debug.ts'
|
|
4
3
|
import { DatasetType } from './exchangedetails.ts'
|
|
4
|
+
import { forEachConcurrent } from './foreachconcurrent.ts'
|
|
5
5
|
import { addDays, doubleDigit, download, parseAsUTCDate, sequence } from './handy.ts'
|
|
6
6
|
import { getOptions } from './options.ts'
|
|
7
7
|
import { Exchange } from './types.ts'
|
|
@@ -62,8 +62,9 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
|
|
|
62
62
|
)
|
|
63
63
|
|
|
64
64
|
// download the rest concurrently up to the CONCURRENCY_LIMIT
|
|
65
|
-
await
|
|
65
|
+
await forEachConcurrent(
|
|
66
66
|
sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
|
|
67
|
+
CONCURRENCY_LIMIT,
|
|
67
68
|
(offset) =>
|
|
68
69
|
downloadDataSet(
|
|
69
70
|
getDownloadOptions({
|
|
@@ -77,8 +78,7 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
|
|
|
77
78
|
date: addDays(startDate, offset)
|
|
78
79
|
}),
|
|
79
80
|
skipIfExists
|
|
80
|
-
)
|
|
81
|
-
{ concurrency: CONCURRENCY_LIMIT }
|
|
81
|
+
)
|
|
82
82
|
)
|
|
83
83
|
const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000
|
|
84
84
|
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export async function forEachConcurrent<T>(items: Iterable<T>, concurrency: number, action: (item: T) => Promise<unknown>) {
|
|
2
|
+
const iterator = items[Symbol.iterator]()
|
|
3
|
+
let hasError = false
|
|
4
|
+
let firstError: unknown
|
|
5
|
+
|
|
6
|
+
const runWorker = async () => {
|
|
7
|
+
while (hasError === false) {
|
|
8
|
+
const next = iterator.next()
|
|
9
|
+
if (next.done) {
|
|
10
|
+
return
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
try {
|
|
14
|
+
await action(next.value)
|
|
15
|
+
} catch (error) {
|
|
16
|
+
if (hasError === false) {
|
|
17
|
+
hasError = true
|
|
18
|
+
firstError = error
|
|
19
|
+
}
|
|
20
|
+
return
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
await Promise.all(Array.from({ length: concurrency }, () => runWorker()))
|
|
26
|
+
if (hasError) {
|
|
27
|
+
throw firstError
|
|
28
|
+
}
|
|
29
|
+
}
|
package/src/worker.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import dbg from 'debug'
|
|
2
2
|
import { existsSync } from 'node:fs'
|
|
3
|
-
import pMap from 'p-map'
|
|
4
3
|
import { isMainThread, parentPort, workerData } from 'worker_threads'
|
|
5
|
-
import {
|
|
4
|
+
import { forEachConcurrent } from './foreachconcurrent.ts'
|
|
5
|
+
import { addMinutes, download, formatDateToPath, optimizeFilters, sha256, wait, cleanTempFiles } from './handy.ts'
|
|
6
6
|
import type { DataFeedCompression } from './options.ts'
|
|
7
7
|
import { Exchange, Filter } from './types.ts'
|
|
8
8
|
const debug = dbg('tardis-dev')
|
|
@@ -123,14 +123,10 @@ async function getAvailableDataFeedSlices(
|
|
|
123
123
|
}
|
|
124
124
|
|
|
125
125
|
// it both begining and end date of the range is accessible fetch all remaning slices concurently with CONCURRENCY_LIMIT
|
|
126
|
-
await
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
await getDataFeedSlice(payload, offset, filters, cacheDir, requestedSliceSize)
|
|
131
|
-
},
|
|
132
|
-
{ concurrency: concurrencyLimit }
|
|
133
|
-
)
|
|
126
|
+
await forEachConcurrent(sliceOffsets, concurrencyLimit, async (offset) => {
|
|
127
|
+
const requestedSliceSize = Math.min(replaySliceSize, minutesCountToFetch - 1 - offset)
|
|
128
|
+
await getDataFeedSlice(payload, offset, filters, cacheDir, requestedSliceSize)
|
|
129
|
+
})
|
|
134
130
|
}
|
|
135
131
|
|
|
136
132
|
async function getDataFeedSlice(
|