tardis-dev 17.3.0 → 18.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boundedqueue.d.ts +19 -0
- package/dist/boundedqueue.d.ts.map +1 -0
- package/dist/boundedqueue.js +96 -0
- package/dist/boundedqueue.js.map +1 -0
- package/dist/combine.d.ts +6 -4
- package/dist/combine.d.ts.map +1 -1
- package/dist/combine.js +9 -34
- package/dist/combine.js.map +1 -1
- package/dist/computable/computable.d.ts.map +1 -1
- package/dist/computable/computable.js +6 -6
- package/dist/computable/computable.js.map +1 -1
- package/dist/downloaddatasets.d.ts.map +1 -1
- package/dist/downloaddatasets.js +4 -4
- package/dist/downloaddatasets.js.map +1 -1
- package/dist/foreachconcurrent.d.ts +2 -0
- package/dist/foreachconcurrent.d.ts.map +1 -0
- package/dist/foreachconcurrent.js +28 -0
- package/dist/foreachconcurrent.js.map +1 -0
- package/dist/realtimefeeds/binance.d.ts.map +1 -1
- package/dist/realtimefeeds/binance.js +9 -0
- package/dist/realtimefeeds/binance.js.map +1 -1
- package/dist/realtimefeeds/binancedex.d.ts.map +1 -1
- package/dist/realtimefeeds/binancedex.js +3 -0
- package/dist/realtimefeeds/binancedex.js.map +1 -1
- package/dist/realtimefeeds/bitstamp.d.ts.map +1 -1
- package/dist/realtimefeeds/bitstamp.js +3 -0
- package/dist/realtimefeeds/bitstamp.js.map +1 -1
- package/dist/realtimefeeds/huobi.d.ts.map +1 -1
- package/dist/realtimefeeds/huobi.js +3 -0
- package/dist/realtimefeeds/huobi.js.map +1 -1
- package/dist/realtimefeeds/index.d.ts +0 -1
- package/dist/realtimefeeds/index.d.ts.map +1 -1
- package/dist/realtimefeeds/index.js +0 -3
- package/dist/realtimefeeds/index.js.map +1 -1
- package/dist/realtimefeeds/kucoin.d.ts.map +1 -1
- package/dist/realtimefeeds/kucoin.js +3 -0
- package/dist/realtimefeeds/kucoin.js.map +1 -1
- package/dist/realtimefeeds/kucoinfutures.d.ts.map +1 -1
- package/dist/realtimefeeds/kucoinfutures.js +3 -0
- package/dist/realtimefeeds/kucoinfutures.js.map +1 -1
- package/dist/realtimefeeds/realtimefeed.d.ts +17 -8
- package/dist/realtimefeeds/realtimefeed.d.ts.map +1 -1
- package/dist/realtimefeeds/realtimefeed.js +186 -111
- package/dist/realtimefeeds/realtimefeed.js.map +1 -1
- package/dist/realtimefeeds/woox.d.ts.map +1 -1
- package/dist/realtimefeeds/woox.js +3 -0
- package/dist/realtimefeeds/woox.js.map +1 -1
- package/dist/realtimeiterator.d.ts +10 -0
- package/dist/realtimeiterator.d.ts.map +1 -0
- package/dist/realtimeiterator.js +106 -0
- package/dist/realtimeiterator.js.map +1 -0
- package/dist/stream.d.ts.map +1 -1
- package/dist/stream.js +50 -21
- package/dist/stream.js.map +1 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +3 -3
- package/dist/worker.js.map +1 -1
- package/package.json +1 -2
- package/src/boundedqueue.ts +118 -0
- package/src/combine.ts +16 -42
- package/src/computable/computable.ts +6 -5
- package/src/downloaddatasets.ts +4 -4
- package/src/foreachconcurrent.ts +29 -0
- package/src/realtimefeeds/binance.ts +10 -0
- package/src/realtimefeeds/binancedex.ts +3 -0
- package/src/realtimefeeds/bitstamp.ts +3 -0
- package/src/realtimefeeds/huobi.ts +4 -0
- package/src/realtimefeeds/index.ts +0 -4
- package/src/realtimefeeds/kucoin.ts +3 -0
- package/src/realtimefeeds/kucoinfutures.ts +3 -0
- package/src/realtimefeeds/realtimefeed.ts +230 -125
- package/src/realtimefeeds/woox.ts +3 -0
- package/src/realtimeiterator.ts +124 -0
- package/src/stream.ts +66 -33
- package/src/worker.ts +6 -10
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { BoundedQueue } from "./boundedqueue.js";
|
|
2
|
+
const REAL_TIME_BUFFER_CAPACITY = 8096;
|
|
3
|
+
const managedRealTimeIterators = new WeakSet();
|
|
4
|
+
export function createManagedRealTimeIterator(iterator, closeSource) {
|
|
5
|
+
const closeSourceOnce = onceAsync(closeSource);
|
|
6
|
+
const returnOnce = onceAsync(async (value) => {
|
|
7
|
+
const sourceClosed = closeSourceOnce();
|
|
8
|
+
const iteratorReturned = iterator.return === undefined ? Promise.resolve({ done: true, value }) : iterator.return(value);
|
|
9
|
+
const [, result] = await Promise.all([sourceClosed, iteratorReturned]);
|
|
10
|
+
return result;
|
|
11
|
+
});
|
|
12
|
+
const managedIterator = {
|
|
13
|
+
[Symbol.asyncIterator]() {
|
|
14
|
+
return this;
|
|
15
|
+
},
|
|
16
|
+
next(value) {
|
|
17
|
+
return iterator.next(value);
|
|
18
|
+
},
|
|
19
|
+
return(value) {
|
|
20
|
+
// Generator return() waits behind a pending next(), so unblock the source first.
|
|
21
|
+
return returnOnce(value);
|
|
22
|
+
},
|
|
23
|
+
async [Symbol.asyncDispose]() {
|
|
24
|
+
await returnOnce();
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
managedRealTimeIterators.add(managedIterator);
|
|
28
|
+
return managedIterator;
|
|
29
|
+
}
|
|
30
|
+
export async function closeIterator(iterator) {
|
|
31
|
+
await iterator.return?.();
|
|
32
|
+
}
|
|
33
|
+
export function isManagedRealTimeIterator(iterator) {
|
|
34
|
+
return (typeof iterator === 'object' && iterator !== null) || typeof iterator === 'function'
|
|
35
|
+
? managedRealTimeIterators.has(iterator)
|
|
36
|
+
: false;
|
|
37
|
+
}
|
|
38
|
+
export function mergeRealTime(sources) {
|
|
39
|
+
const output = new BoundedQueue(REAL_TIME_BUFFER_CAPACITY);
|
|
40
|
+
const iterators = sources.map((source) => source[Symbol.asyncIterator]());
|
|
41
|
+
const closeSources = onceAsync(async () => {
|
|
42
|
+
output.close();
|
|
43
|
+
await Promise.allSettled(iterators.map(closeIterator));
|
|
44
|
+
});
|
|
45
|
+
return createManagedRealTimeIterator(merge(iterators, output, closeSources), closeSources);
|
|
46
|
+
}
|
|
47
|
+
async function* merge(iterators, output, closeSources) {
|
|
48
|
+
let activeWriters = iterators.length;
|
|
49
|
+
const writers = iterators.map(async (iterator) => {
|
|
50
|
+
try {
|
|
51
|
+
while (output.isOpen) {
|
|
52
|
+
const result = await iterator.next();
|
|
53
|
+
if (result.done) {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
const written = output.write(result.value);
|
|
57
|
+
if (written !== true && (await written) === false) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
if (output.isOpen) {
|
|
64
|
+
output.fail(error instanceof Error ? error : new Error(String(error)));
|
|
65
|
+
void closeSources();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
finally {
|
|
69
|
+
activeWriters--;
|
|
70
|
+
if (activeWriters === 0) {
|
|
71
|
+
output.finish();
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
if (writers.length === 0) {
|
|
76
|
+
output.finish();
|
|
77
|
+
}
|
|
78
|
+
try {
|
|
79
|
+
while (true) {
|
|
80
|
+
const result = await output.next();
|
|
81
|
+
if (result.done) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
yield result.value;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
await closeSources();
|
|
89
|
+
await Promise.allSettled(writers);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
function onceAsync(callback) {
|
|
93
|
+
let promise;
|
|
94
|
+
return (...args) => {
|
|
95
|
+
if (promise === undefined) {
|
|
96
|
+
try {
|
|
97
|
+
promise = Promise.resolve(callback(...args));
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
promise = Promise.reject(error);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return promise;
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
//# sourceMappingURL=realtimeiterator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"realtimeiterator.js","sourceRoot":"","sources":["../src/realtimeiterator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,MAAM,yBAAyB,GAAG,IAAI,CAAA;AACtC,MAAM,wBAAwB,GAAG,IAAI,OAAO,EAAU,CAAA;AAQtD,MAAM,UAAU,6BAA6B,CAC3C,QAAkC,EAClC,WAA0B;IAE1B,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAC,CAAA;IAC9C,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,EAAE,KAAW,EAAE,EAAE;QACjD,MAAM,YAAY,GAAG,eAAe,EAAE,CAAA;QACtC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAa,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACjI,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC,CAAA;QACtE,OAAO,MAAM,CAAA;IACf,CAAC,CAAC,CAAA;IAEF,MAAM,eAAe,GAA+B;QAClD,CAAC,MAAM,CAAC,aAAa,CAAC;YACpB,OAAO,IAAI,CAAA;QACb,CAAC;QACD,IAAI,CAAC,KAAW;YACd,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC7B,CAAC;QACD,MAAM,CAAC,KAAW;YAChB,iFAAiF;YACjF,OAAO,UAAU,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;QACD,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;YACzB,MAAM,UAAU,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;IAED,wBAAwB,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IAC7C,OAAO,eAAe,CAAA;AACxB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgC;IAClE,MAAM,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,QAAiB;IACzD,OAAO,CAAC,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,IAAI,OAAO,QAAQ,KAAK,UAAU;QAC1F,CAAC,CAAC,wBAAwB,CAAC,GAAG,CAAC,QAAQ,CAAC;QACxC,CAAC,CAAC,KAAK,CAAA;AACX,CAAC;AAED,MAAM,UAAU,aAAa,CAAI,OAA2B;IAC1D,MAAM,MAAM,GAAG,IAAI,YAAY,CAAI,yBAAyB,CAAC,CAAA;IAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;IACzE,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,IAAI,EAAE;QACxC,MAAM,CAAC,KAAK,EAAE,CAAA;QACd,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;IACxD,CAAC,CAAC,CAAA;IAEF,OAAO,6BAA6B,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,YAAY,CAAC,CAAA;AAC5F,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,KAAK,CAAI,SAA6B,EAAE,MAAuB,EAAE,YAAoC;IACnH,IAAI,aAAa,GAAG,SAAS,CAAC,MAAM,CAAA;IAEpC,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC/C,IAAI,CAAC;YACH,OAAO,MAAM,CAAC,MAAM,EAAE,CAAC;gBACrB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBACpC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBAChB,OAAM;gBACR,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;gBAC1C,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,MAAM,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC;oBAClD,OAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;gBACtE,KAAK,YAAY,EAAE,CAAA;YACrB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,aAAa,EAAE,CAAA;YACf,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,MAAM,EAAE,CAAA;YACjB,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,CAAC,MAAM,EAAE,CAAA;IACjB,CAAC;IAED,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;YAClC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,OAAM;YACR,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAAA;QACpB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,YAAY,EAAE,CAAA;QACpB,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;IACnC,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAmC,QAAqC;IACxF,IAAI,OAA8C,CAAA;IAClD,OAAO,CAAC,GAAG,IAAW,EAAE,EAAE;QACxB,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;YAC9C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACjC,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC,CAAA;AACH,CAAC"}
|
package/dist/stream.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../src/stream.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;
|
|
1
|
+
{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../src/stream.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAGlD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAU,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAE5E,KAAK,YAAY,CAAC,CAAC,IAAI,CAAC,SAAS,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;AACxE,KAAK,uBAAuB,CAAC,CAAC,SAAS,SAAS,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,SAAS,OAAO,IAAI,CAAC,SAAS,IAAI,GAC1G,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,UAAU,GACpC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;AAkD3B,wBAAgB,MAAM,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,EAAE,EACpE,QAAQ,EACR,OAAO,EACP,iBAAyB,EACzB,eAA2B,EAC3B,OAAmB,EACpB,EAAE,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,qBAAqB,CAC5C,CAAC,SAAS,IAAI,GAAG;IAAE,cAAc,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,GAAG,SAAS,GAAG;IAAE,cAAc,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,CAC7G,CASA;AA+ED,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,IAAI;IACzE,QAAQ,EAAE,CAAC,CAAA;IACX,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAA;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,eAAe,CAAC,EAAE,CAAC,CAAA;IACnB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;CACjC,CAAA;AAED,MAAM,MAAM,uBAAuB,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,IAAI;IACnF,QAAQ,EAAE,CAAC,CAAA;IACX,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,sBAAsB,CAAC,EAAE,CAAC,CAAA;IAC1B,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;CACjC,CAAA;AAED,wBAAgB,gBAAgB,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,EAC/G,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAyB,EAAE,sBAAkC,EAAE,OAAmB,EAAE,EAAE,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,EACxI,GAAG,WAAW,EAAE,CAAC,GAChB,qBAAqB,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAUtD"}
|
package/dist/stream.js
CHANGED
|
@@ -1,43 +1,64 @@
|
|
|
1
1
|
import { debug } from "./debug.js";
|
|
2
2
|
import { createNormalizedSymbolFilter, getFilters, normalizeMessages } from "./handy.js";
|
|
3
|
+
import { createManagedRealTimeIterator } from "./realtimeiterator.js";
|
|
3
4
|
import { createRealTimeFeed } from "./realtimefeeds/index.js";
|
|
4
|
-
async function* _stream({ exchange, filters, timeoutIntervalMS = 10000, withDisconnects = undefined, onError = undefined }) {
|
|
5
|
+
async function* _stream({ exchange, filters, timeoutIntervalMS = 10000, withDisconnects = undefined, onError = undefined }, state) {
|
|
5
6
|
validateStreamOptions(filters);
|
|
6
7
|
const realTimeFeed = createRealTimeFeed(exchange, filters, timeoutIntervalMS, onError);
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
8
|
+
const messages = realTimeFeed[Symbol.asyncIterator]();
|
|
9
|
+
state.activeIterator = messages;
|
|
10
|
+
try {
|
|
11
|
+
while (true) {
|
|
12
|
+
const result = await messages.next();
|
|
13
|
+
if (result.done) {
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
const message = result.value;
|
|
17
|
+
if (message.__disconnect__ === true) {
|
|
18
|
+
// __disconnect__ message means that websocket connection has been closed
|
|
19
|
+
// notify about it by yielding undefined if flag is set
|
|
20
|
+
if (withDisconnects) {
|
|
21
|
+
yield undefined;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
yield {
|
|
26
|
+
localTimestamp: new Date(),
|
|
27
|
+
message
|
|
28
|
+
};
|
|
13
29
|
}
|
|
14
30
|
}
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
31
|
+
}
|
|
32
|
+
finally {
|
|
33
|
+
if (state.activeIterator === messages) {
|
|
34
|
+
state.activeIterator = undefined;
|
|
35
|
+
await messages.return?.();
|
|
20
36
|
}
|
|
21
37
|
}
|
|
22
38
|
}
|
|
23
39
|
export function stream({ exchange, filters, timeoutIntervalMS = 10000, withDisconnects = undefined, onError = undefined }) {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
return
|
|
40
|
+
const state = {};
|
|
41
|
+
const iterator = _stream({ exchange, filters, timeoutIntervalMS, withDisconnects, onError }, state);
|
|
42
|
+
return createManagedRealTimeIterator(iterator, () => {
|
|
43
|
+
const activeIterator = state.activeIterator;
|
|
44
|
+
state.activeIterator = undefined;
|
|
45
|
+
return activeIterator?.return?.();
|
|
46
|
+
});
|
|
27
47
|
}
|
|
28
|
-
async function* _streamNormalized({ exchange, symbols, timeoutIntervalMS = 10000, withDisconnectMessages = undefined, onError = undefined }, ...normalizers) {
|
|
29
|
-
while (
|
|
48
|
+
async function* _streamNormalized({ exchange, symbols, timeoutIntervalMS = 10000, withDisconnectMessages = undefined, onError = undefined }, state, ...normalizers) {
|
|
49
|
+
while (state.closed === false) {
|
|
30
50
|
try {
|
|
31
51
|
const createMappers = (localTimestamp) => normalizers.map((m) => m(exchange, localTimestamp));
|
|
32
52
|
const mappers = createMappers(new Date());
|
|
33
53
|
const filters = getFilters(mappers, symbols);
|
|
34
|
-
const messages =
|
|
54
|
+
const messages = stream({
|
|
35
55
|
exchange,
|
|
36
56
|
withDisconnects: true,
|
|
37
57
|
timeoutIntervalMS,
|
|
38
58
|
filters,
|
|
39
59
|
onError
|
|
40
60
|
});
|
|
61
|
+
state.activeIterator = messages;
|
|
41
62
|
// filter normalized messages by symbol as some exchanges do not offer subscribing to specific symbols for some of the channels
|
|
42
63
|
// for example Phemex market24h channel
|
|
43
64
|
const filter = createNormalizedSymbolFilter(symbols, filters);
|
|
@@ -47,6 +68,9 @@ async function* _streamNormalized({ exchange, symbols, timeoutIntervalMS = 10000
|
|
|
47
68
|
}
|
|
48
69
|
}
|
|
49
70
|
catch (error) {
|
|
71
|
+
if (state.closed) {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
50
74
|
if (onError !== undefined) {
|
|
51
75
|
onError(error);
|
|
52
76
|
}
|
|
@@ -76,8 +100,13 @@ function validateStreamOptions(filters) {
|
|
|
76
100
|
}
|
|
77
101
|
}
|
|
78
102
|
export function streamNormalized({ exchange, symbols, timeoutIntervalMS = 10000, withDisconnectMessages = undefined, onError = undefined }, ...normalizers) {
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
return
|
|
103
|
+
const state = { closed: false };
|
|
104
|
+
const iterator = _streamNormalized({ exchange, symbols, timeoutIntervalMS, withDisconnectMessages, onError }, state, ...normalizers);
|
|
105
|
+
return createManagedRealTimeIterator(iterator, () => {
|
|
106
|
+
state.closed = true;
|
|
107
|
+
const activeIterator = state.activeIterator;
|
|
108
|
+
state.activeIterator = undefined;
|
|
109
|
+
return activeIterator?.return?.();
|
|
110
|
+
});
|
|
82
111
|
}
|
|
83
112
|
//# sourceMappingURL=stream.js.map
|
package/dist/stream.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stream.js","sourceRoot":"","sources":["../src/stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAClC,OAAO,EAAE,4BAA4B,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAExF,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;
|
|
1
|
+
{"version":3,"file":"stream.js","sourceRoot":"","sources":["../src/stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAClC,OAAO,EAAE,4BAA4B,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAExF,OAAO,EAAE,6BAA6B,EAAgC,MAAM,uBAAuB,CAAA;AACnG,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAiB7D,KAAK,SAAS,CAAC,CAAC,OAAO,CACrB,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,GAAG,KAAK,EAAE,eAAe,GAAG,SAAS,EAAE,OAAO,GAAG,SAAS,EAAuB,EACvH,KAAkB;IAElB,qBAAqB,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,YAAY,GAAG,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAA;IACtF,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;IACrD,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAA;IAE/B,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;YACpC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,OAAM;YACR,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAA;YAC5B,IAAI,OAAO,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;gBACpC,yEAAyE;gBACzE,uDAAuD;gBACvD,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,SAAgB,CAAA;gBACxB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM;oBACJ,cAAc,EAAE,IAAI,IAAI,EAAE;oBAC1B,OAAO;iBACD,CAAA;YACV,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,KAAK,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;YACtC,KAAK,CAAC,cAAc,GAAG,SAAS,CAAA;YAChC,MAAM,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAA;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAgD,EACpE,QAAQ,EACR,OAAO,EACP,iBAAiB,GAAG,KAAK,EACzB,eAAe,GAAG,SAAS,EAC3B,OAAO,GAAG,SAAS,EACC;IAGpB,MAAM,KAAK,GAAgB,EAAE,CAAA;IAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,CAAA;IAEnG,OAAO,6BAA6B,CAAC,QAAQ,EAAE,GAAG,EAAE;QAClD,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,CAAA;QAC3C,KAAK,CAAC,cAAc,GAAG,SAAS,CAAA;QAChC,OAAO,cAAc,EAAE,MAAM,EAAE,EAAE,CAAA;IACnC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,iBAAiB,CAC/B,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,GAAG,KAAK,EAAE,sBAAsB,GAAG,SAAS,EAAE,OAAO,GAAG,SAAS,EAAiC,EACxI,KAA4B,EAC5B,GAAG,WAAc;IAEjB,OAAO,KAAK,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,CAAC,cAAoB,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAA;YACnG,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAE5C,MAAM,QAAQ,GAAG,MAAM,CAAC;gBACtB,QAAQ;gBACR,eAAe,EAAE,IAAI;gBACrB,iBAAiB;gBACjB,OAAO;gBACP,OAAO;aACR,CAAiC,CAAA;YAClC,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAA;YAE/B,+HAA+H;YAC/H,uCAAuC;YAEvC,MAAM,MAAM,GAAG,4BAA4B,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAE7D,MAAM,kBAAkB,GAAG,iBAAiB,CAC1C,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,OAAO,EACP,aAAa,EACb,sBAAsB,EACtB,MAAM,EACN,IAAI,IAAI,EAAE,CACX,CAAA;YAED,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;gBAC/C,MAAM,OAAO,CAAA;YACf,CAAC;QACH,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACjB,OAAM;YACR,CAAC;YAED,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,OAAO,CAAC,KAAK,CAAC,CAAA;YAChB,CAAC;YACD,KAAK,CAAC,kEAAkE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAA;YAC1F,IAAI,sBAAsB,EAAE,CAAC;gBAC3B,gDAAgD;gBAChD,MAAM,UAAU,GAAe;oBAC7B,IAAI,EAAE,YAAY;oBAClB,QAAQ;oBACR,cAAc,EAAE,IAAI,IAAI,EAAE;oBAC1B,OAAO;iBACR,CAAA;gBAED,MAAM,UAAiB,CAAA;YACzB,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAyB;IACtD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;IAC7E,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QAEzB,IAAI,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,OAAO,0CAA0C,CAAC,CAAA;QACpH,CAAC;IACH,CAAC;AACH,CAAC;AAkBD,MAAM,UAAU,gBAAgB,CAC9B,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,GAAG,KAAK,EAAE,sBAAsB,GAAG,SAAS,EAAE,OAAO,GAAG,SAAS,EAAiC,EACxI,GAAG,WAAc;IAEjB,MAAM,KAAK,GAA0B,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IACtD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,GAAG,WAAW,CAAC,CAAA;IAEpI,OAAO,6BAA6B,CAAC,QAAQ,EAAE,GAAG,EAAE;QAClD,KAAK,CAAC,MAAM,GAAG,IAAI,CAAA;QACnB,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,CAAA;QAC3C,KAAK,CAAC,cAAc,GAAG,SAAS,CAAA;QAChC,OAAO,cAAc,EAAE,MAAM,EAAE,EAAE,CAAA;IACnC,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
package/dist/worker.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAgM7C,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,mBAAmB,EAAE,mBAAmB,CAAA;IACxC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,IAAI,CAAA;IACd,MAAM,EAAE,IAAI,CAAA;IACZ,QAAQ,EAAE,QAAQ,CAAA;IAClB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IACtB,2BAA2B,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAC/C,CAAA;AAED,0BAAkB,YAAY;IAC5B,gBAAgB,qBAAqB;IACrC,kBAAkB,uBAAuB;CAC1C"}
|
package/dist/worker.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import dbg from 'debug';
|
|
2
2
|
import { existsSync } from 'node:fs';
|
|
3
|
-
import pMap from 'p-map';
|
|
4
3
|
import { isMainThread, parentPort, workerData } from 'worker_threads';
|
|
4
|
+
import { forEachConcurrent } from "./foreachconcurrent.js";
|
|
5
5
|
import { addMinutes, download, formatDateToPath, optimizeFilters, sha256, wait, cleanTempFiles } from "./handy.js";
|
|
6
6
|
const debug = dbg('tardis-dev');
|
|
7
7
|
const DEFAULT_DATA_FEED_SLICE_SIZE = 1;
|
|
@@ -82,10 +82,10 @@ async function getAvailableDataFeedSlices(payload, filters, cacheDir, minutesCou
|
|
|
82
82
|
sliceOffsets.push(offset);
|
|
83
83
|
}
|
|
84
84
|
// it both begining and end date of the range is accessible fetch all remaning slices concurently with CONCURRENCY_LIMIT
|
|
85
|
-
await
|
|
85
|
+
await forEachConcurrent(sliceOffsets, concurrencyLimit, async (offset) => {
|
|
86
86
|
const requestedSliceSize = Math.min(replaySliceSize, minutesCountToFetch - 1 - offset);
|
|
87
87
|
await getDataFeedSlice(payload, offset, filters, cacheDir, requestedSliceSize);
|
|
88
|
-
}
|
|
88
|
+
});
|
|
89
89
|
}
|
|
90
90
|
async function getDataFeedSlice({ exchange, fromDate, endpoint, apiKey, dataFeedCompression, userAgent }, offset, filters, cacheDir, requestedSliceSize = DEFAULT_DATA_FEED_SLICE_SIZE, useCache = true) {
|
|
91
91
|
const sliceTimestamp = addMinutes(fromDate, offset);
|
package/dist/worker.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,OAAO,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,OAAO,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAGlH,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,CAAA;AAE/B,MAAM,4BAA4B,GAAG,CAAC,CAAA;AAEtC,IAAI,YAAY,EAAE,CAAC;IACjB,KAAK,CAAC,mDAAmD,CAAC,CAAA;AAC5D,CAAC;KAAM,CAAC;IACN,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,MAAoB,EAAE,EAAE;QACjD,IAAI,MAAM,2DAAkC,EAAE,CAAC;YAC7C,cAAc,EAAE,CAAA;YAChB,UAAW,CAAC,WAAW,4DAAiC,CAAA;QAC1D,CAAC;IACH,CAAC,CAAC,CAAA;IACF,iBAAiB,CAAC,UAA8B,CAAC,CAAA;AACnD,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;IAChD,KAAK,CAAC,wCAAwC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;IAC7D,MAAM,GAAG,CAAA;AACX,CAAC,CAAC,CAAA;AAEF,KAAK,UAAU,iBAAiB,CAAC,OAAyB;IACxD,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,CAAA;IACxC,MAAM,uCAAuC,GAAG,CAAC,CAAA;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAA;IAC5B,mEAAmE;IACnE,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEhD,2HAA2H;IAC3H,qEAAqE;IACrE,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,sBAAsB,CAAC,CAAA;IAExH,gEAAgE;IAChE,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,QAAQ,UAAU,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;IAEnF,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,2BAA2B,KAAK,QAAQ;QACrD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,2BAA2B,EAAE,uCAAuC,CAAC;QACxF,CAAC,CAAC,EAAE,CAAA;IACR,MAAM,YAAY,GAAG,iBAAiB,GAAG,sBAAsB,CAAA;IAE/D,MAAM,0CAA0C,GAAG,MAAM,wBAAwB,CAC/E,OAAO,EACP,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,CACvB,CAAA;IAED,MAAM,0BAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,0CAA0C,EAAE,iBAAiB,CAAC,CAAA;IAE3H,2DAA2D;IAC3D,KAAK,IAAI,MAAM,GAAG,0CAA0C,EAAE,MAAM,GAAG,mBAAmB,EAAE,MAAM,EAAE,EAAE,CAAC;QACrG,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,GAAG,sBAAsB,CAAA;QACrF,MAAM,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;QAEzE,IAAI,gBAAgB,GAAG,6BAA6B,EAAE,CAAC;YACrD,MAAM,QAAQ,GAAG,gBAAgB,GAAG,6BAA6B,GAAG,GAAG,CAAA;YAEvE,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtB,CAAC;QACD,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAA;IAC5D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,OAAyB,EACzB,mBAA2B,EAC3B,YAAoB,EACpB,oBAA4B;IAE5B,MAAM,0BAA0B,GAAG,OAAO,CAAC,2BAA2B,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACxI,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAChC,OAAO,mBAAmB,CAAA;IAC5B,CAAC;IAED,IAAI,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IAEvE,4EAA4E;IAC5E,IAAI,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/D,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,6BAA6B,CAAA;QAClF,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,CAAA;QAC7B,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACnE,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,oBAAoB,CAAC,CAAA;IAC7H,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAA;AAC1E,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,OAAyB,EACzB,OAAiB,EACjB,QAAgB,EAChB,mBAA2B,EAC3B,gBAAwB;IAExB,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAM;IACR,CAAC;IAED,iHAAiH;IACjH,qEAAqE;IACrE,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAElI,kGAAkG;IAClG,MAAM,UAAU,GACd,mBAAmB,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAEpI,MAAM,eAAe,GACnB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB,CAAC,CAAA;IAC7H,MAAM,YAAY,GAAa,EAAE,CAAA;IACjC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,mBAAmB,GAAG,CAAC,EAAE,MAAM,IAAI,eAAe,EAAE,CAAC;QACjF,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC;IAED,wHAAwH;IACxH,MAAM,iBAAiB,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACvE,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,mBAAmB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAA;QACtF,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;IAChF,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAoB,EAC1F,MAAc,EACd,OAAiB,EACjB,QAAgB,EAChB,kBAAkB,GAAG,4BAA4B,EACjD,QAAQ,GAAG,IAAI;IAEf,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,eAAe,GAAG,kBAAkB,KAAK,4BAA4B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,kBAAkB,EAAE,CAAA;IAChH,MAAM,aAAa,GAAG,GAAG,QAAQ,IAAI,gBAAgB,CAAC,cAAc,CAAC,GAAG,eAAe,OAAO,CAAA;IAC9F,MAAM,aAAa,GAAG,GAAG,aAAa,MAAM,CAAA;IAC5C,MAAM,aAAa,GAAG,GAAG,aAAa,KAAK,CAAA;IAC3C,IAAI,eAAe,CAAA;IACnB,IAAI,QAAQ,EAAE,CAAC;QACb,eAAe,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAA;IACrH,CAAC;IAED,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,KAAK,CAAC,oDAAoD,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;QACzF,MAAM,OAAO,GAAkB;YAC7B,QAAQ;YACR,SAAS,EAAE,eAAe;YAC1B,SAAS,EAAE,kBAAkB;SAC9B,CAAA;QACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QAChC,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,kBAAkB,EAAE,4BAA4B;SACjD,CAAA;IACH,CAAC;IAED,IAAI,GAAG,GAAG,GAAG,QAAQ,eAAe,QAAQ,SAAS,QAAQ,CAAC,WAAW,EAAE,WAAW,MAAM,gBAAgB,mBAAmB,EAAE,CAAA;IACjI,IAAI,kBAAkB,GAAG,4BAA4B,EAAE,CAAC;QACtD,GAAG,IAAI,cAAc,kBAAkB,EAAE,CAAA;IAC3C,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,IAAI,YAAY,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC;QACpC,MAAM;QACN,YAAY,EAAE,aAAa;QAC3B,GAAG;QACH,SAAS;QACT,8BAA8B,EAAE,IAAI;QACpC,cAAc,EAAE,mBAAmB,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY;KACvE,CAAC,CAAA;IACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAA;IACxE,MAAM,kBAAkB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,wBAAwB,CAAC,IAAI,4BAA4B,CAAC,CAAA;IAEnH,KAAK,CAAC,iEAAiE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAA;IACrG,MAAM,OAAO,GAAkB;QAC7B,QAAQ;QACR,SAAS,EAAE,cAAc,CAAC,YAAY;QACtC,SAAS,EAAE,iBAAiB;KAC7B,CAAA;IACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;IAEhC,OAAO;QACL,SAAS,EAAE,iBAAiB;QAC5B,kBAAkB;KACnB,CAAA;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tardis-dev",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "18.0.1",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=24.5"
|
|
6
6
|
},
|
|
@@ -70,7 +70,6 @@
|
|
|
70
70
|
"dependencies": {
|
|
71
71
|
"debug": "^4.4.3",
|
|
72
72
|
"follow-redirects": "^1.16.0",
|
|
73
|
-
"p-map": "^7.0.5",
|
|
74
73
|
"protobufjs": "^8.7.1",
|
|
75
74
|
"ws": "^8.21.1"
|
|
76
75
|
},
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
type PendingRead<T> = {
|
|
2
|
+
resolve: (result: IteratorResult<T>) => void
|
|
3
|
+
reject: (error: Error) => void
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
type PendingWrite<T> = {
|
|
7
|
+
value: T
|
|
8
|
+
resolve: (written: boolean) => void
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export class BoundedQueue<T> {
|
|
12
|
+
private readonly values: (T | undefined)[]
|
|
13
|
+
private readIndex = 0
|
|
14
|
+
private size = 0
|
|
15
|
+
private pendingWrites: PendingWrite<T>[] = []
|
|
16
|
+
private pendingRead?: PendingRead<T>
|
|
17
|
+
private failure?: Error
|
|
18
|
+
private open = true
|
|
19
|
+
|
|
20
|
+
constructor(private readonly capacity: number) {
|
|
21
|
+
this.values = new Array(capacity)
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
get isOpen() {
|
|
25
|
+
return this.open
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
write(value: T): boolean | Promise<boolean> {
|
|
29
|
+
if (!this.open) {
|
|
30
|
+
return false
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
if (this.pendingRead !== undefined) {
|
|
34
|
+
const { resolve } = this.pendingRead
|
|
35
|
+
this.pendingRead = undefined
|
|
36
|
+
resolve({ done: false, value })
|
|
37
|
+
return true
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
if (this.size < this.capacity) {
|
|
41
|
+
this.values[(this.readIndex + this.size) % this.capacity] = value
|
|
42
|
+
this.size++
|
|
43
|
+
return true
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return new Promise((resolve) => this.pendingWrites.push({ value, resolve }))
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async next(): Promise<IteratorResult<T>> {
|
|
50
|
+
if (this.size > 0) {
|
|
51
|
+
const value = this.values[this.readIndex] as T
|
|
52
|
+
this.values[this.readIndex] = undefined
|
|
53
|
+
this.readIndex = (this.readIndex + 1) % this.capacity
|
|
54
|
+
this.size--
|
|
55
|
+
const pendingWrite = this.pendingWrites.shift()
|
|
56
|
+
if (pendingWrite !== undefined) {
|
|
57
|
+
this.values[(this.readIndex + this.size) % this.capacity] = pendingWrite.value
|
|
58
|
+
this.size++
|
|
59
|
+
pendingWrite.resolve(true)
|
|
60
|
+
}
|
|
61
|
+
return { done: false, value }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (this.failure !== undefined) {
|
|
65
|
+
throw this.failure
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (!this.open) {
|
|
69
|
+
return { done: true, value: undefined }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return new Promise((resolve, reject) => {
|
|
73
|
+
this.pendingRead = { resolve, reject }
|
|
74
|
+
})
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
finish() {
|
|
78
|
+
if (!this.open) {
|
|
79
|
+
return
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
this.open = false
|
|
83
|
+
this.pendingRead?.resolve({ done: true, value: undefined })
|
|
84
|
+
this.pendingRead = undefined
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
fail(error: Error) {
|
|
88
|
+
if (!this.open) {
|
|
89
|
+
return
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
this.failure = error
|
|
93
|
+
this.stop(error)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
close() {
|
|
97
|
+
this.stop()
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
private stop(error?: Error) {
|
|
101
|
+
this.open = false
|
|
102
|
+
this.values.fill(undefined)
|
|
103
|
+
this.readIndex = 0
|
|
104
|
+
this.size = 0
|
|
105
|
+
for (const { resolve } of this.pendingWrites) {
|
|
106
|
+
resolve(false)
|
|
107
|
+
}
|
|
108
|
+
this.pendingWrites = []
|
|
109
|
+
if (this.pendingRead !== undefined) {
|
|
110
|
+
if (error === undefined) {
|
|
111
|
+
this.pendingRead.resolve({ done: true, value: undefined })
|
|
112
|
+
} else {
|
|
113
|
+
this.pendingRead.reject(error)
|
|
114
|
+
}
|
|
115
|
+
this.pendingRead = undefined
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
package/src/combine.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { once } from 'events'
|
|
1
|
+
import { closeIterator, isManagedRealTimeIterator, mergeRealTime } from './realtimeiterator.ts'
|
|
3
2
|
|
|
4
3
|
type NextMessageResultWithIndex = {
|
|
5
4
|
index: number
|
|
@@ -11,6 +10,12 @@ type Combinable = { localTimestamp: Date }
|
|
|
11
10
|
const DATE_MAX = new Date(8640000000000000)
|
|
12
11
|
|
|
13
12
|
type OffsetMS = number | ((message: Combinable) => number)
|
|
13
|
+
type IteratorPayload = AsyncIterableIterator<Combinable>[] | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS }[]
|
|
14
|
+
type CombinedMessage<T> = T extends AsyncIterableIterator<infer U>[]
|
|
15
|
+
? U
|
|
16
|
+
: T extends { stream: AsyncIterableIterator<infer U> }[]
|
|
17
|
+
? U
|
|
18
|
+
: never
|
|
14
19
|
|
|
15
20
|
async function nextWithIndex(
|
|
16
21
|
iterator: AsyncIterableIterator<Combinable> | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS },
|
|
@@ -71,56 +76,27 @@ function findOldestResult(oldest: NextMessageResultWithIndex, current: NextMessa
|
|
|
71
76
|
|
|
72
77
|
// combines multiple iterators from for example multiple exchanges
|
|
73
78
|
// works both for real-time and historical data
|
|
74
|
-
export
|
|
75
|
-
T extends AsyncIterableIterator<Combinable>[] | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS }[]
|
|
76
|
-
>(
|
|
77
|
-
...iteratorsPayload: T
|
|
78
|
-
): AsyncIterableIterator<
|
|
79
|
-
T extends AsyncIterableIterator<infer U>[] ? U : T extends { stream: AsyncIterableIterator<infer Z> }[] ? Z : never
|
|
80
|
-
> {
|
|
79
|
+
export function combine<T extends IteratorPayload>(...iteratorsPayload: T): AsyncIterableIterator<CombinedMessage<T>> {
|
|
81
80
|
const iterators = iteratorsPayload.map((payload) => {
|
|
82
81
|
if ('stream' in payload) {
|
|
83
82
|
return payload.stream
|
|
84
83
|
}
|
|
85
84
|
return payload
|
|
86
85
|
})
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
return
|
|
86
|
+
if (isManagedRealTimeIterator(iterators[0])) {
|
|
87
|
+
return mergeRealTime(iterators) as AsyncIterableIterator<CombinedMessage<T>>
|
|
90
88
|
}
|
|
91
|
-
// decide based on first provided iterator if we're dealing with real-time or historical data streams
|
|
92
|
-
if ((iterators[0] as any).__realtime__) {
|
|
93
|
-
const combinedStream = new PassThrough({
|
|
94
|
-
objectMode: true,
|
|
95
|
-
highWaterMark: 8096
|
|
96
|
-
})
|
|
97
|
-
|
|
98
|
-
combinedStream.setMaxListeners(iterators.length + 1)
|
|
99
|
-
|
|
100
|
-
iterators.forEach(async function writeMessagesToCombinedStream(messages) {
|
|
101
|
-
for await (const message of messages) {
|
|
102
|
-
if (combinedStream.destroyed) {
|
|
103
|
-
return
|
|
104
|
-
}
|
|
105
89
|
|
|
106
|
-
|
|
107
|
-
// Handle backpressure on write
|
|
108
|
-
await once(combinedStream, 'drain')
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
})
|
|
112
|
-
|
|
113
|
-
for await (const message of combinedStream) {
|
|
114
|
-
yield message
|
|
115
|
-
}
|
|
116
|
-
} else {
|
|
117
|
-
return yield* combineHistorical(iteratorsPayload) as any
|
|
118
|
-
}
|
|
90
|
+
return combineHistorical(iteratorsPayload) as AsyncIterableIterator<CombinedMessage<T>>
|
|
119
91
|
}
|
|
120
92
|
|
|
121
93
|
async function* combineHistorical(
|
|
122
94
|
iterators: AsyncIterableIterator<Combinable>[] | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS }[]
|
|
123
95
|
) {
|
|
96
|
+
if (iterators.length === 0) {
|
|
97
|
+
return
|
|
98
|
+
}
|
|
99
|
+
|
|
124
100
|
try {
|
|
125
101
|
// wait for all results to resolve
|
|
126
102
|
const results = await Promise.all(iterators.map(nextWithIndex))
|
|
@@ -153,8 +129,6 @@ async function* combineHistorical(
|
|
|
153
129
|
}
|
|
154
130
|
} while (aliveIteratorsCount > 0)
|
|
155
131
|
} finally {
|
|
156
|
-
|
|
157
|
-
;(iterator as any).return()
|
|
158
|
-
}
|
|
132
|
+
await Promise.all(iterators.map((iterator) => closeIterator('stream' in iterator ? iterator.stream : iterator)))
|
|
159
133
|
}
|
|
160
134
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Disconnect, Exchange, NormalizedData } from '../types.ts'
|
|
2
|
+
import { closeIterator, createManagedRealTimeIterator, isManagedRealTimeIterator } from '../realtimeiterator.ts'
|
|
2
3
|
import { ComputableContext, createWithContext } from './context.ts'
|
|
3
4
|
|
|
4
5
|
export type Computable<T extends NormalizedData> = {
|
|
@@ -39,7 +40,7 @@ async function* _compute<T extends ComputableFactory<any>[], U extends Normalize
|
|
|
39
40
|
}
|
|
40
41
|
}
|
|
41
42
|
} finally {
|
|
42
|
-
messages
|
|
43
|
+
await closeIterator(messages)
|
|
43
44
|
}
|
|
44
45
|
}
|
|
45
46
|
|
|
@@ -47,13 +48,13 @@ export function compute<T extends ComputableFactory<any>[], U extends Normalized
|
|
|
47
48
|
messages: AsyncIterableIterator<U>,
|
|
48
49
|
...computables: T
|
|
49
50
|
): AsyncIterableIterator<T extends ComputableFactory<infer Z>[] ? (U extends Disconnect ? U | Z | Disconnect : U | Z) : never> {
|
|
50
|
-
|
|
51
|
+
const iterator = _compute(messages, ...computables)
|
|
51
52
|
|
|
52
|
-
if ((messages
|
|
53
|
-
|
|
53
|
+
if (isManagedRealTimeIterator(messages)) {
|
|
54
|
+
return createManagedRealTimeIterator(iterator, () => closeIterator(messages))
|
|
54
55
|
}
|
|
55
56
|
|
|
56
|
-
return
|
|
57
|
+
return iterator
|
|
57
58
|
}
|
|
58
59
|
|
|
59
60
|
class Computables {
|
package/src/downloaddatasets.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs'
|
|
2
|
-
import pMap from 'p-map'
|
|
3
2
|
import { debug } from './debug.ts'
|
|
4
3
|
import { DatasetType } from './exchangedetails.ts'
|
|
4
|
+
import { forEachConcurrent } from './foreachconcurrent.ts'
|
|
5
5
|
import { addDays, doubleDigit, download, parseAsUTCDate, sequence } from './handy.ts'
|
|
6
6
|
import { getOptions } from './options.ts'
|
|
7
7
|
import { Exchange } from './types.ts'
|
|
@@ -62,8 +62,9 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
|
|
|
62
62
|
)
|
|
63
63
|
|
|
64
64
|
// download the rest concurrently up to the CONCURRENCY_LIMIT
|
|
65
|
-
await
|
|
65
|
+
await forEachConcurrent(
|
|
66
66
|
sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
|
|
67
|
+
CONCURRENCY_LIMIT,
|
|
67
68
|
(offset) =>
|
|
68
69
|
downloadDataSet(
|
|
69
70
|
getDownloadOptions({
|
|
@@ -77,8 +78,7 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
|
|
|
77
78
|
date: addDays(startDate, offset)
|
|
78
79
|
}),
|
|
79
80
|
skipIfExists
|
|
80
|
-
)
|
|
81
|
-
{ concurrency: CONCURRENCY_LIMIT }
|
|
81
|
+
)
|
|
82
82
|
)
|
|
83
83
|
const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000
|
|
84
84
|
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export async function forEachConcurrent<T>(items: Iterable<T>, concurrency: number, action: (item: T) => Promise<unknown>) {
|
|
2
|
+
const iterator = items[Symbol.iterator]()
|
|
3
|
+
let hasError = false
|
|
4
|
+
let firstError: unknown
|
|
5
|
+
|
|
6
|
+
const runWorker = async () => {
|
|
7
|
+
while (hasError === false) {
|
|
8
|
+
const next = iterator.next()
|
|
9
|
+
if (next.done) {
|
|
10
|
+
return
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
try {
|
|
14
|
+
await action(next.value)
|
|
15
|
+
} catch (error) {
|
|
16
|
+
if (hasError === false) {
|
|
17
|
+
hasError = true
|
|
18
|
+
firstError = error
|
|
19
|
+
}
|
|
20
|
+
return
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
await Promise.all(Array.from({ length: concurrency }, () => runWorker()))
|
|
26
|
+
if (hasError) {
|
|
27
|
+
throw firstError
|
|
28
|
+
}
|
|
29
|
+
}
|