tardis-dev 17.3.0 → 18.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/boundedqueue.d.ts +19 -0
  2. package/dist/boundedqueue.d.ts.map +1 -0
  3. package/dist/boundedqueue.js +96 -0
  4. package/dist/boundedqueue.js.map +1 -0
  5. package/dist/combine.d.ts +6 -4
  6. package/dist/combine.d.ts.map +1 -1
  7. package/dist/combine.js +9 -34
  8. package/dist/combine.js.map +1 -1
  9. package/dist/computable/computable.d.ts.map +1 -1
  10. package/dist/computable/computable.js +6 -6
  11. package/dist/computable/computable.js.map +1 -1
  12. package/dist/downloaddatasets.d.ts.map +1 -1
  13. package/dist/downloaddatasets.js +4 -4
  14. package/dist/downloaddatasets.js.map +1 -1
  15. package/dist/foreachconcurrent.d.ts +2 -0
  16. package/dist/foreachconcurrent.d.ts.map +1 -0
  17. package/dist/foreachconcurrent.js +28 -0
  18. package/dist/foreachconcurrent.js.map +1 -0
  19. package/dist/realtimefeeds/binance.d.ts.map +1 -1
  20. package/dist/realtimefeeds/binance.js +9 -0
  21. package/dist/realtimefeeds/binance.js.map +1 -1
  22. package/dist/realtimefeeds/binancedex.d.ts.map +1 -1
  23. package/dist/realtimefeeds/binancedex.js +3 -0
  24. package/dist/realtimefeeds/binancedex.js.map +1 -1
  25. package/dist/realtimefeeds/bitstamp.d.ts.map +1 -1
  26. package/dist/realtimefeeds/bitstamp.js +3 -0
  27. package/dist/realtimefeeds/bitstamp.js.map +1 -1
  28. package/dist/realtimefeeds/huobi.d.ts.map +1 -1
  29. package/dist/realtimefeeds/huobi.js +3 -0
  30. package/dist/realtimefeeds/huobi.js.map +1 -1
  31. package/dist/realtimefeeds/index.d.ts +0 -1
  32. package/dist/realtimefeeds/index.d.ts.map +1 -1
  33. package/dist/realtimefeeds/index.js +0 -3
  34. package/dist/realtimefeeds/index.js.map +1 -1
  35. package/dist/realtimefeeds/kucoin.d.ts.map +1 -1
  36. package/dist/realtimefeeds/kucoin.js +3 -0
  37. package/dist/realtimefeeds/kucoin.js.map +1 -1
  38. package/dist/realtimefeeds/kucoinfutures.d.ts.map +1 -1
  39. package/dist/realtimefeeds/kucoinfutures.js +3 -0
  40. package/dist/realtimefeeds/kucoinfutures.js.map +1 -1
  41. package/dist/realtimefeeds/realtimefeed.d.ts +17 -8
  42. package/dist/realtimefeeds/realtimefeed.d.ts.map +1 -1
  43. package/dist/realtimefeeds/realtimefeed.js +186 -111
  44. package/dist/realtimefeeds/realtimefeed.js.map +1 -1
  45. package/dist/realtimefeeds/woox.d.ts.map +1 -1
  46. package/dist/realtimefeeds/woox.js +3 -0
  47. package/dist/realtimefeeds/woox.js.map +1 -1
  48. package/dist/realtimeiterator.d.ts +10 -0
  49. package/dist/realtimeiterator.d.ts.map +1 -0
  50. package/dist/realtimeiterator.js +106 -0
  51. package/dist/realtimeiterator.js.map +1 -0
  52. package/dist/stream.d.ts.map +1 -1
  53. package/dist/stream.js +50 -21
  54. package/dist/stream.js.map +1 -1
  55. package/dist/worker.d.ts.map +1 -1
  56. package/dist/worker.js +3 -3
  57. package/dist/worker.js.map +1 -1
  58. package/package.json +1 -2
  59. package/src/boundedqueue.ts +118 -0
  60. package/src/combine.ts +16 -42
  61. package/src/computable/computable.ts +6 -5
  62. package/src/downloaddatasets.ts +4 -4
  63. package/src/foreachconcurrent.ts +29 -0
  64. package/src/realtimefeeds/binance.ts +10 -0
  65. package/src/realtimefeeds/binancedex.ts +3 -0
  66. package/src/realtimefeeds/bitstamp.ts +3 -0
  67. package/src/realtimefeeds/huobi.ts +4 -0
  68. package/src/realtimefeeds/index.ts +0 -4
  69. package/src/realtimefeeds/kucoin.ts +3 -0
  70. package/src/realtimefeeds/kucoinfutures.ts +3 -0
  71. package/src/realtimefeeds/realtimefeed.ts +230 -125
  72. package/src/realtimefeeds/woox.ts +3 -0
  73. package/src/realtimeiterator.ts +124 -0
  74. package/src/stream.ts +66 -33
  75. package/src/worker.ts +6 -10
@@ -0,0 +1,106 @@
1
+ import { BoundedQueue } from "./boundedqueue.js";
2
+ const REAL_TIME_BUFFER_CAPACITY = 8096;
3
+ const managedRealTimeIterators = new WeakSet();
4
+ export function createManagedRealTimeIterator(iterator, closeSource) {
5
+ const closeSourceOnce = onceAsync(closeSource);
6
+ const returnOnce = onceAsync(async (value) => {
7
+ const sourceClosed = closeSourceOnce();
8
+ const iteratorReturned = iterator.return === undefined ? Promise.resolve({ done: true, value }) : iterator.return(value);
9
+ const [, result] = await Promise.all([sourceClosed, iteratorReturned]);
10
+ return result;
11
+ });
12
+ const managedIterator = {
13
+ [Symbol.asyncIterator]() {
14
+ return this;
15
+ },
16
+ next(value) {
17
+ return iterator.next(value);
18
+ },
19
+ return(value) {
20
+ // Generator return() waits behind a pending next(), so unblock the source first.
21
+ return returnOnce(value);
22
+ },
23
+ async [Symbol.asyncDispose]() {
24
+ await returnOnce();
25
+ }
26
+ };
27
+ managedRealTimeIterators.add(managedIterator);
28
+ return managedIterator;
29
+ }
30
+ export async function closeIterator(iterator) {
31
+ await iterator.return?.();
32
+ }
33
+ export function isManagedRealTimeIterator(iterator) {
34
+ return (typeof iterator === 'object' && iterator !== null) || typeof iterator === 'function'
35
+ ? managedRealTimeIterators.has(iterator)
36
+ : false;
37
+ }
38
+ export function mergeRealTime(sources) {
39
+ const output = new BoundedQueue(REAL_TIME_BUFFER_CAPACITY);
40
+ const iterators = sources.map((source) => source[Symbol.asyncIterator]());
41
+ const closeSources = onceAsync(async () => {
42
+ output.close();
43
+ await Promise.allSettled(iterators.map(closeIterator));
44
+ });
45
+ return createManagedRealTimeIterator(merge(iterators, output, closeSources), closeSources);
46
+ }
47
+ async function* merge(iterators, output, closeSources) {
48
+ let activeWriters = iterators.length;
49
+ const writers = iterators.map(async (iterator) => {
50
+ try {
51
+ while (output.isOpen) {
52
+ const result = await iterator.next();
53
+ if (result.done) {
54
+ return;
55
+ }
56
+ const written = output.write(result.value);
57
+ if (written !== true && (await written) === false) {
58
+ return;
59
+ }
60
+ }
61
+ }
62
+ catch (error) {
63
+ if (output.isOpen) {
64
+ output.fail(error instanceof Error ? error : new Error(String(error)));
65
+ void closeSources();
66
+ }
67
+ }
68
+ finally {
69
+ activeWriters--;
70
+ if (activeWriters === 0) {
71
+ output.finish();
72
+ }
73
+ }
74
+ });
75
+ if (writers.length === 0) {
76
+ output.finish();
77
+ }
78
+ try {
79
+ while (true) {
80
+ const result = await output.next();
81
+ if (result.done) {
82
+ return;
83
+ }
84
+ yield result.value;
85
+ }
86
+ }
87
+ finally {
88
+ await closeSources();
89
+ await Promise.allSettled(writers);
90
+ }
91
+ }
92
+ function onceAsync(callback) {
93
+ let promise;
94
+ return (...args) => {
95
+ if (promise === undefined) {
96
+ try {
97
+ promise = Promise.resolve(callback(...args));
98
+ }
99
+ catch (error) {
100
+ promise = Promise.reject(error);
101
+ }
102
+ }
103
+ return promise;
104
+ };
105
+ }
106
+ //# sourceMappingURL=realtimeiterator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"realtimeiterator.js","sourceRoot":"","sources":["../src/realtimeiterator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,MAAM,yBAAyB,GAAG,IAAI,CAAA;AACtC,MAAM,wBAAwB,GAAG,IAAI,OAAO,EAAU,CAAA;AAQtD,MAAM,UAAU,6BAA6B,CAC3C,QAAkC,EAClC,WAA0B;IAE1B,MAAM,eAAe,GAAG,SAAS,CAAC,WAAW,CAAC,CAAA;IAC9C,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,EAAE,KAAW,EAAE,EAAE;QACjD,MAAM,YAAY,GAAG,eAAe,EAAE,CAAA;QACtC,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAa,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACjI,MAAM,CAAC,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC,CAAA;QACtE,OAAO,MAAM,CAAA;IACf,CAAC,CAAC,CAAA;IAEF,MAAM,eAAe,GAA+B;QAClD,CAAC,MAAM,CAAC,aAAa,CAAC;YACpB,OAAO,IAAI,CAAA;QACb,CAAC;QACD,IAAI,CAAC,KAAW;YACd,OAAO,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC7B,CAAC;QACD,MAAM,CAAC,KAAW;YAChB,iFAAiF;YACjF,OAAO,UAAU,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;QACD,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;YACzB,MAAM,UAAU,EAAE,CAAA;QACpB,CAAC;KACF,CAAA;IAED,wBAAwB,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IAC7C,OAAO,eAAe,CAAA;AACxB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgC;IAClE,MAAM,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,QAAiB;IACzD,OAAO,CAAC,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,IAAI,OAAO,QAAQ,KAAK,UAAU;QAC1F,CAAC,CAAC,wBAAwB,CAAC,GAAG,CAAC,QAAQ,CAAC;QACxC,CAAC,CAAC,KAAK,CAAA;AACX,CAAC;AAED,MAAM,UAAU,aAAa,CAAI,OAA2B;IAC1D,MAAM,MAAM,GAAG,IAAI,YAAY,CAAI,yBAAyB,CAAC,CAAA;IAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;IACzE,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,IAAI,EAAE;QACxC,MAAM,CAAC,KAAK,EAAE,CAAA;QACd,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;IACxD,CAAC,CAAC,CAAA;IAEF,OAAO,6BAA6B,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,YAAY,CAAC,CAAA;AAC5F,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,KAAK,CAAI,SAA6B,EAAE,MAAuB,EAAE,YAAoC;IACnH,IAAI,aAAa,GAAG,SAAS,CAAC,MAAM,CAAA;IAEpC,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC/C,IAAI,CAAC;YACH,OAAO,MAAM,CAAC,MAAM,EAAE,CAAC;gBACrB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;gBACpC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;oBAChB,OAAM;gBACR,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;gBAC1C,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,MAAM,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC;oBAClD,OAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;gBACtE,KAAK,YAAY,EAAE,CAAA;YACrB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,aAAa,EAAE,CAAA;YACf,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,MAAM,EAAE,CAAA;YACjB,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,CAAC,MAAM,EAAE,CAAA;IACjB,CAAC;IAED,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;YAClC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,OAAM;YACR,CAAC;YACD,MAAM,MAAM,CAAC,KAAK,CAAA;QACpB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,YAAY,EAAE,CAAA;QACpB,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAA;IACnC,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAmC,QAAqC;IACxF,IAAI,OAA8C,CAAA;IAClD,OAAO,CAAC,GAAG,IAAW,EAAE,EAAE;QACxB,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,CAAA;YAC9C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACjC,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC,CAAA;AACH,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../src/stream.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAElD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAU,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAE5E,KAAK,YAAY,CAAC,CAAC,IAAI,CAAC,SAAS,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;AACxE,KAAK,uBAAuB,CAAC,CAAC,SAAS,SAAS,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,SAAS,OAAO,IAAI,CAAC,SAAS,IAAI,GAC1G,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,UAAU,GACpC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;AA+B3B,wBAAgB,MAAM,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,EAAE,EACpE,QAAQ,EACR,OAAO,EACP,iBAAyB,EACzB,eAA2B,EAC3B,OAAmB,EACpB,EAAE,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,qBAAqB,CAC5C,CAAC,SAAS,IAAI,GAAG;IAAE,cAAc,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,GAAG,SAAS,GAAG;IAAE,cAAc,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,CAC7G,CAMA;AAyED,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,IAAI;IACzE,QAAQ,EAAE,CAAC,CAAA;IACX,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAA;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,eAAe,CAAC,EAAE,CAAC,CAAA;IACnB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;CACjC,CAAA;AAED,MAAM,MAAM,uBAAuB,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,IAAI;IACnF,QAAQ,EAAE,CAAC,CAAA;IACX,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,sBAAsB,CAAC,EAAE,CAAC,CAAA;IAC1B,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;CACjC,CAAA;AAED,wBAAgB,gBAAgB,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,EAC/G,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAyB,EAAE,sBAAkC,EAAE,OAAmB,EAAE,EAAE,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,EACxI,GAAG,WAAW,EAAE,CAAC,GAChB,qBAAqB,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAMtD"}
1
+ {"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../src/stream.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAGlD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAU,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAE5E,KAAK,YAAY,CAAC,CAAC,IAAI,CAAC,SAAS,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;AACxE,KAAK,uBAAuB,CAAC,CAAC,SAAS,SAAS,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,SAAS,OAAO,IAAI,CAAC,SAAS,IAAI,GAC1G,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,UAAU,GACpC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;AAkD3B,wBAAgB,MAAM,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,EAAE,EACpE,QAAQ,EACR,OAAO,EACP,iBAAyB,EACzB,eAA2B,EAC3B,OAAmB,EACpB,EAAE,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,qBAAqB,CAC5C,CAAC,SAAS,IAAI,GAAG;IAAE,cAAc,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,GAAG,SAAS,GAAG;IAAE,cAAc,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,CAC7G,CASA;AA+ED,MAAM,MAAM,aAAa,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,IAAI;IACzE,QAAQ,EAAE,CAAC,CAAA;IACX,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAA;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,eAAe,CAAC,EAAE,CAAC,CAAA;IACnB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;CACjC,CAAA;AAED,MAAM,MAAM,uBAAuB,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,IAAI;IACnF,QAAQ,EAAE,CAAC,CAAA;IACX,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,sBAAsB,CAAC,EAAE,CAAC,CAAA;IAC1B,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;CACjC,CAAA;AAED,wBAAgB,gBAAgB,CAAC,CAAC,SAAS,QAAQ,EAAE,CAAC,SAAS,aAAa,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,SAAS,OAAO,GAAG,KAAK,EAC/G,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAyB,EAAE,sBAAkC,EAAE,OAAmB,EAAE,EAAE,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,EACxI,GAAG,WAAW,EAAE,CAAC,GAChB,qBAAqB,CAAC,uBAAuB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAUtD"}
package/dist/stream.js CHANGED
@@ -1,43 +1,64 @@
1
1
  import { debug } from "./debug.js";
2
2
  import { createNormalizedSymbolFilter, getFilters, normalizeMessages } from "./handy.js";
3
+ import { createManagedRealTimeIterator } from "./realtimeiterator.js";
3
4
  import { createRealTimeFeed } from "./realtimefeeds/index.js";
4
- async function* _stream({ exchange, filters, timeoutIntervalMS = 10000, withDisconnects = undefined, onError = undefined }) {
5
+ async function* _stream({ exchange, filters, timeoutIntervalMS = 10000, withDisconnects = undefined, onError = undefined }, state) {
5
6
  validateStreamOptions(filters);
6
7
  const realTimeFeed = createRealTimeFeed(exchange, filters, timeoutIntervalMS, onError);
7
- for await (const message of realTimeFeed) {
8
- if (message.__disconnect__ === true) {
9
- // __disconnect__ message means that websocket connection has been closed
10
- // notify about it by yielding undefined if flag is set
11
- if (withDisconnects) {
12
- yield undefined;
8
+ const messages = realTimeFeed[Symbol.asyncIterator]();
9
+ state.activeIterator = messages;
10
+ try {
11
+ while (true) {
12
+ const result = await messages.next();
13
+ if (result.done) {
14
+ return;
15
+ }
16
+ const message = result.value;
17
+ if (message.__disconnect__ === true) {
18
+ // __disconnect__ message means that websocket connection has been closed
19
+ // notify about it by yielding undefined if flag is set
20
+ if (withDisconnects) {
21
+ yield undefined;
22
+ }
23
+ }
24
+ else {
25
+ yield {
26
+ localTimestamp: new Date(),
27
+ message
28
+ };
13
29
  }
14
30
  }
15
- else {
16
- yield {
17
- localTimestamp: new Date(),
18
- message
19
- };
31
+ }
32
+ finally {
33
+ if (state.activeIterator === messages) {
34
+ state.activeIterator = undefined;
35
+ await messages.return?.();
20
36
  }
21
37
  }
22
38
  }
23
39
  export function stream({ exchange, filters, timeoutIntervalMS = 10000, withDisconnects = undefined, onError = undefined }) {
24
- let _iterator = _stream({ exchange, filters, timeoutIntervalMS, withDisconnects, onError });
25
- _iterator.__realtime__ = true;
26
- return _iterator;
40
+ const state = {};
41
+ const iterator = _stream({ exchange, filters, timeoutIntervalMS, withDisconnects, onError }, state);
42
+ return createManagedRealTimeIterator(iterator, () => {
43
+ const activeIterator = state.activeIterator;
44
+ state.activeIterator = undefined;
45
+ return activeIterator?.return?.();
46
+ });
27
47
  }
28
- async function* _streamNormalized({ exchange, symbols, timeoutIntervalMS = 10000, withDisconnectMessages = undefined, onError = undefined }, ...normalizers) {
29
- while (true) {
48
+ async function* _streamNormalized({ exchange, symbols, timeoutIntervalMS = 10000, withDisconnectMessages = undefined, onError = undefined }, state, ...normalizers) {
49
+ while (state.closed === false) {
30
50
  try {
31
51
  const createMappers = (localTimestamp) => normalizers.map((m) => m(exchange, localTimestamp));
32
52
  const mappers = createMappers(new Date());
33
53
  const filters = getFilters(mappers, symbols);
34
- const messages = _stream({
54
+ const messages = stream({
35
55
  exchange,
36
56
  withDisconnects: true,
37
57
  timeoutIntervalMS,
38
58
  filters,
39
59
  onError
40
60
  });
61
+ state.activeIterator = messages;
41
62
  // filter normalized messages by symbol as some exchanges do not offer subscribing to specific symbols for some of the channels
42
63
  // for example Phemex market24h channel
43
64
  const filter = createNormalizedSymbolFilter(symbols, filters);
@@ -47,6 +68,9 @@ async function* _streamNormalized({ exchange, symbols, timeoutIntervalMS = 10000
47
68
  }
48
69
  }
49
70
  catch (error) {
71
+ if (state.closed) {
72
+ return;
73
+ }
50
74
  if (onError !== undefined) {
51
75
  onError(error);
52
76
  }
@@ -76,8 +100,13 @@ function validateStreamOptions(filters) {
76
100
  }
77
101
  }
78
102
  export function streamNormalized({ exchange, symbols, timeoutIntervalMS = 10000, withDisconnectMessages = undefined, onError = undefined }, ...normalizers) {
79
- let _iterator = _streamNormalized({ exchange, symbols, timeoutIntervalMS, withDisconnectMessages, onError }, ...normalizers);
80
- _iterator.__realtime__ = true;
81
- return _iterator;
103
+ const state = { closed: false };
104
+ const iterator = _streamNormalized({ exchange, symbols, timeoutIntervalMS, withDisconnectMessages, onError }, state, ...normalizers);
105
+ return createManagedRealTimeIterator(iterator, () => {
106
+ state.closed = true;
107
+ const activeIterator = state.activeIterator;
108
+ state.activeIterator = undefined;
109
+ return activeIterator?.return?.();
110
+ });
82
111
  }
83
112
  //# sourceMappingURL=stream.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"stream.js","sourceRoot":"","sources":["../src/stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAClC,OAAO,EAAE,4BAA4B,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAExF,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAQ7D,KAAK,SAAS,CAAC,CAAC,OAAO,CAAgD,EACrE,QAAQ,EACR,OAAO,EACP,iBAAiB,GAAG,KAAK,EACzB,eAAe,GAAG,SAAS,EAC3B,OAAO,GAAG,SAAS,EACC;IAGpB,qBAAqB,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,YAAY,GAAG,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAA;IAEtF,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;QACzC,IAAI,OAAO,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;YACpC,yEAAyE;YACzE,uDAAuD;YACvD,IAAI,eAAe,EAAE,CAAC;gBACpB,MAAM,SAAgB,CAAA;YACxB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM;gBACJ,cAAc,EAAE,IAAI,IAAI,EAAE;gBAC1B,OAAO;aACD,CAAA;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAgD,EACpE,QAAQ,EACR,OAAO,EACP,iBAAiB,GAAG,KAAK,EACzB,eAAe,GAAG,SAAS,EAC3B,OAAO,GAAG,SAAS,EACC;IAGpB,IAAI,SAAS,GAAG,OAAO,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC,CAE1F;IAAC,SAAiB,CAAC,YAAY,GAAG,IAAI,CAAA;IAEvC,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,iBAAiB,CAC/B,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,GAAG,KAAK,EAAE,sBAAsB,GAAG,SAAS,EAAE,OAAO,GAAG,SAAS,EAAiC,EACxI,GAAG,WAAc;IAEjB,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,CAAC,cAAoB,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAA;YACnG,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAE5C,MAAM,QAAQ,GAAG,OAAO,CAAC;gBACvB,QAAQ;gBACR,eAAe,EAAE,IAAI;gBACrB,iBAAiB;gBACjB,OAAO;gBACP,OAAO;aACR,CAAC,CAAA;YAEF,+HAA+H;YAC/H,uCAAuC;YAEvC,MAAM,MAAM,GAAG,4BAA4B,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAE7D,MAAM,kBAAkB,GAAG,iBAAiB,CAC1C,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,OAAO,EACP,aAAa,EACb,sBAAsB,EACtB,MAAM,EACN,IAAI,IAAI,EAAE,CACX,CAAA;YAED,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;gBAC/C,MAAM,OAAO,CAAA;YACf,CAAC;QACH,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,OAAO,CAAC,KAAK,CAAC,CAAA;YAChB,CAAC;YACD,KAAK,CAAC,kEAAkE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAA;YAC1F,IAAI,sBAAsB,EAAE,CAAC;gBAC3B,gDAAgD;gBAChD,MAAM,UAAU,GAAe;oBAC7B,IAAI,EAAE,YAAY;oBAClB,QAAQ;oBACR,cAAc,EAAE,IAAI,IAAI,EAAE;oBAC1B,OAAO;iBACR,CAAA;gBAED,MAAM,UAAiB,CAAA;YACzB,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAyB;IACtD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;IAC7E,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QAEzB,IAAI,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,OAAO,0CAA0C,CAAC,CAAA;QACpH,CAAC;IACH,CAAC;AACH,CAAC;AAkBD,MAAM,UAAU,gBAAgB,CAC9B,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,GAAG,KAAK,EAAE,sBAAsB,GAAG,SAAS,EAAE,OAAO,GAAG,SAAS,EAAiC,EACxI,GAAG,WAAc;IAEjB,IAAI,SAAS,GAAG,iBAAiB,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,OAAO,EAAE,EAAE,GAAG,WAAW,CAAC,CAE3H;IAAC,SAAiB,CAAC,YAAY,GAAG,IAAI,CAAA;IAEvC,OAAO,SAAS,CAAA;AAClB,CAAC"}
1
+ {"version":3,"file":"stream.js","sourceRoot":"","sources":["../src/stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAA;AAClC,OAAO,EAAE,4BAA4B,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAExF,OAAO,EAAE,6BAA6B,EAAgC,MAAM,uBAAuB,CAAA;AACnG,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAiB7D,KAAK,SAAS,CAAC,CAAC,OAAO,CACrB,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,GAAG,KAAK,EAAE,eAAe,GAAG,SAAS,EAAE,OAAO,GAAG,SAAS,EAAuB,EACvH,KAAkB;IAElB,qBAAqB,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,YAAY,GAAG,kBAAkB,CAAC,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAA;IACtF,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAA;IACrD,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAA;IAE/B,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;YACpC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,OAAM;YACR,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAA;YAC5B,IAAI,OAAO,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;gBACpC,yEAAyE;gBACzE,uDAAuD;gBACvD,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,SAAgB,CAAA;gBACxB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM;oBACJ,cAAc,EAAE,IAAI,IAAI,EAAE;oBAC1B,OAAO;iBACD,CAAA;YACV,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,KAAK,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;YACtC,KAAK,CAAC,cAAc,GAAG,SAAS,CAAA;YAChC,MAAM,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAA;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,MAAM,CAAgD,EACpE,QAAQ,EACR,OAAO,EACP,iBAAiB,GAAG,KAAK,EACzB,eAAe,GAAG,SAAS,EAC3B,OAAO,GAAG,SAAS,EACC;IAGpB,MAAM,KAAK,GAAgB,EAAE,CAAA;IAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,CAAA;IAEnG,OAAO,6BAA6B,CAAC,QAAQ,EAAE,GAAG,EAAE;QAClD,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,CAAA;QAC3C,KAAK,CAAC,cAAc,GAAG,SAAS,CAAA;QAChC,OAAO,cAAc,EAAE,MAAM,EAAE,EAAE,CAAA;IACnC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,iBAAiB,CAC/B,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,GAAG,KAAK,EAAE,sBAAsB,GAAG,SAAS,EAAE,OAAO,GAAG,SAAS,EAAiC,EACxI,KAA4B,EAC5B,GAAG,WAAc;IAEjB,OAAO,KAAK,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,CAAC,cAAoB,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAA;YACnG,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAE5C,MAAM,QAAQ,GAAG,MAAM,CAAC;gBACtB,QAAQ;gBACR,eAAe,EAAE,IAAI;gBACrB,iBAAiB;gBACjB,OAAO;gBACP,OAAO;aACR,CAAiC,CAAA;YAClC,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAA;YAE/B,+HAA+H;YAC/H,uCAAuC;YAEvC,MAAM,MAAM,GAAG,4BAA4B,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAE7D,MAAM,kBAAkB,GAAG,iBAAiB,CAC1C,QAAQ,EACR,OAAO,EACP,QAAQ,EACR,OAAO,EACP,aAAa,EACb,sBAAsB,EACtB,MAAM,EACN,IAAI,IAAI,EAAE,CACX,CAAA;YAED,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;gBAC/C,MAAM,OAAO,CAAA;YACf,CAAC;QACH,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACjB,OAAM;YACR,CAAC;YAED,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,OAAO,CAAC,KAAK,CAAC,CAAA;YAChB,CAAC;YACD,KAAK,CAAC,kEAAkE,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAA;YAC1F,IAAI,sBAAsB,EAAE,CAAC;gBAC3B,gDAAgD;gBAChD,MAAM,UAAU,GAAe;oBAC7B,IAAI,EAAE,YAAY;oBAClB,QAAQ;oBACR,cAAc,EAAE,IAAI,IAAI,EAAE;oBAC1B,OAAO;iBACR,CAAA;gBAED,MAAM,UAAiB,CAAA;YACzB,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAAC,OAAyB;IACtD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;IAC7E,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;QAEzB,IAAI,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,yCAAyC,MAAM,CAAC,OAAO,0CAA0C,CAAC,CAAA;QACpH,CAAC;IACH,CAAC;AACH,CAAC;AAkBD,MAAM,UAAU,gBAAgB,CAC9B,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,GAAG,KAAK,EAAE,sBAAsB,GAAG,SAAS,EAAE,OAAO,GAAG,SAAS,EAAiC,EACxI,GAAG,WAAc;IAEjB,MAAM,KAAK,GAA0B,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IACtD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,GAAG,WAAW,CAAC,CAAA;IAEpI,OAAO,6BAA6B,CAAC,QAAQ,EAAE,GAAG,EAAE;QAClD,KAAK,CAAC,MAAM,GAAG,IAAI,CAAA;QACnB,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,CAAA;QAC3C,KAAK,CAAC,cAAc,GAAG,SAAS,CAAA;QAChC,OAAO,cAAc,EAAE,MAAM,EAAE,EAAE,CAAA;IACnC,CAAC,CAAC,CAAA;AACJ,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAoM7C,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,mBAAmB,EAAE,mBAAmB,CAAA;IACxC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,IAAI,CAAA;IACd,MAAM,EAAE,IAAI,CAAA;IACZ,QAAQ,EAAE,QAAQ,CAAA;IAClB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IACtB,2BAA2B,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAC/C,CAAA;AAED,0BAAkB,YAAY;IAC5B,gBAAgB,qBAAqB;IACrC,kBAAkB,uBAAuB;CAC1C"}
1
+ {"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAgM7C,MAAM,MAAM,aAAa,GAAG;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;CAClB,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,mBAAmB,EAAE,mBAAmB,CAAA;IACxC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,IAAI,CAAA;IACd,MAAM,EAAE,IAAI,CAAA;IACZ,QAAQ,EAAE,QAAQ,CAAA;IAClB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IACtB,2BAA2B,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAC/C,CAAA;AAED,0BAAkB,YAAY;IAC5B,gBAAgB,qBAAqB;IACrC,kBAAkB,uBAAuB;CAC1C"}
package/dist/worker.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import dbg from 'debug';
2
2
  import { existsSync } from 'node:fs';
3
- import pMap from 'p-map';
4
3
  import { isMainThread, parentPort, workerData } from 'worker_threads';
4
+ import { forEachConcurrent } from "./foreachconcurrent.js";
5
5
  import { addMinutes, download, formatDateToPath, optimizeFilters, sha256, wait, cleanTempFiles } from "./handy.js";
6
6
  const debug = dbg('tardis-dev');
7
7
  const DEFAULT_DATA_FEED_SLICE_SIZE = 1;
@@ -82,10 +82,10 @@ async function getAvailableDataFeedSlices(payload, filters, cacheDir, minutesCou
82
82
  sliceOffsets.push(offset);
83
83
  }
84
84
  // it both begining and end date of the range is accessible fetch all remaning slices concurently with CONCURRENCY_LIMIT
85
- await pMap(sliceOffsets, async (offset) => {
85
+ await forEachConcurrent(sliceOffsets, concurrencyLimit, async (offset) => {
86
86
  const requestedSliceSize = Math.min(replaySliceSize, minutesCountToFetch - 1 - offset);
87
87
  await getDataFeedSlice(payload, offset, filters, cacheDir, requestedSliceSize);
88
- }, { concurrency: concurrencyLimit });
88
+ });
89
89
  }
90
90
  async function getDataFeedSlice({ exchange, fromDate, endpoint, apiKey, dataFeedCompression, userAgent }, offset, filters, cacheDir, requestedSliceSize = DEFAULT_DATA_FEED_SLICE_SIZE, useCache = true) {
91
91
  const sliceTimestamp = addMinutes(fromDate, offset);
@@ -1 +1 @@
1
- {"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,OAAO,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,IAAI,MAAM,OAAO,CAAA;AACxB,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AACrE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,eAAe,EAAY,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAG5H,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,CAAA;AAE/B,MAAM,4BAA4B,GAAG,CAAC,CAAA;AAEtC,IAAI,YAAY,EAAE,CAAC;IACjB,KAAK,CAAC,mDAAmD,CAAC,CAAA;AAC5D,CAAC;KAAM,CAAC;IACN,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,MAAoB,EAAE,EAAE;QACjD,IAAI,MAAM,2DAAkC,EAAE,CAAC;YAC7C,cAAc,EAAE,CAAA;YAChB,UAAW,CAAC,WAAW,4DAAiC,CAAA;QAC1D,CAAC;IACH,CAAC,CAAC,CAAA;IACF,iBAAiB,CAAC,UAA8B,CAAC,CAAA;AACnD,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;IAChD,KAAK,CAAC,wCAAwC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;IAC7D,MAAM,GAAG,CAAA;AACX,CAAC,CAAC,CAAA;AAEF,KAAK,UAAU,iBAAiB,CAAC,OAAyB;IACxD,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,CAAA;IACxC,MAAM,uCAAuC,GAAG,CAAC,CAAA;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAA;IAC5B,mEAAmE;IACnE,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEhD,2HAA2H;IAC3H,qEAAqE;IACrE,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,sBAAsB,CAAC,CAAA;IAExH,gEAAgE;IAChE,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,QAAQ,UAAU,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;IAEnF,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,2BAA2B,KAAK,QAAQ;QACrD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,2BAA2B,EAAE,uCAAuC,CAAC;QACxF,CAAC,CAAC,EAAE,CAAA;IACR,MAAM,YAAY,GAAG,iBAAiB,GAAG,sBAAsB,CAAA;IAE/D,MAAM,0CAA0C,GAAG,MAAM,wBAAwB,CAC/E,OAAO,EACP,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,CACvB,CAAA;IAED,MAAM,0BAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,0CAA0C,EAAE,iBAAiB,CAAC,CAAA;IAE3H,2DAA2D;IAC3D,KAAK,IAAI,MAAM,GAAG,0CAA0C,EAAE,MAAM,GAAG,mBAAmB,EAAE,MAAM,EAAE,EAAE,CAAC;QACrG,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,GAAG,sBAAsB,CAAA;QACrF,MAAM,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;QAEzE,IAAI,gBAAgB,GAAG,6BAA6B,EAAE,CAAC;YACrD,MAAM,QAAQ,GAAG,gBAAgB,GAAG,6BAA6B,GAAG,GAAG,CAAA;YAEvE,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtB,CAAC;QACD,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAA;IAC5D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,OAAyB,EACzB,mBAA2B,EAC3B,YAAoB,EACpB,oBAA4B;IAE5B,MAAM,0BAA0B,GAAG,OAAO,CAAC,2BAA2B,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACxI,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAChC,OAAO,mBAAmB,CAAA;IAC5B,CAAC;IAED,IAAI,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IAEvE,4EAA4E;IAC5E,IAAI,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/D,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,6BAA6B,CAAA;QAClF,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,CAAA;QAC7B,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACnE,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,oBAAoB,CAAC,CAAA;IAC7H,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAA;AAC1E,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,OAAyB,EACzB,OAAiB,EACjB,QAAgB,EAChB,mBAA2B,EAC3B,gBAAwB;IAExB,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAM;IACR,CAAC;IAED,iHAAiH;IACjH,qEAAqE;IACrE,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAElI,kGAAkG;IAClG,MAAM,UAAU,GACd,mBAAmB,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAEpI,MAAM,eAAe,GACnB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB,CAAC,CAAA;IAC7H,MAAM,YAAY,GAAa,EAAE,CAAA;IACjC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,mBAAmB,GAAG,CAAC,EAAE,MAAM,IAAI,eAAe,EAAE,CAAC;QACjF,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC;IAED,wHAAwH;IACxH,MAAM,IAAI,CACR,YAAY,EACZ,KAAK,EAAE,MAAM,EAAE,EAAE;QACf,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,mBAAmB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAA;QACtF,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;IAChF,CAAC,EACD,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAClC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAoB,EAC1F,MAAc,EACd,OAAiB,EACjB,QAAgB,EAChB,kBAAkB,GAAG,4BAA4B,EACjD,QAAQ,GAAG,IAAI;IAEf,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,eAAe,GAAG,kBAAkB,KAAK,4BAA4B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,kBAAkB,EAAE,CAAA;IAChH,MAAM,aAAa,GAAG,GAAG,QAAQ,IAAI,gBAAgB,CAAC,cAAc,CAAC,GAAG,eAAe,OAAO,CAAA;IAC9F,MAAM,aAAa,GAAG,GAAG,aAAa,MAAM,CAAA;IAC5C,MAAM,aAAa,GAAG,GAAG,aAAa,KAAK,CAAA;IAC3C,IAAI,eAAe,CAAA;IACnB,IAAI,QAAQ,EAAE,CAAC;QACb,eAAe,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAA;IACrH,CAAC;IAED,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,KAAK,CAAC,oDAAoD,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;QACzF,MAAM,OAAO,GAAkB;YAC7B,QAAQ;YACR,SAAS,EAAE,eAAe;YAC1B,SAAS,EAAE,kBAAkB;SAC9B,CAAA;QACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QAChC,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,kBAAkB,EAAE,4BAA4B;SACjD,CAAA;IACH,CAAC;IAED,IAAI,GAAG,GAAG,GAAG,QAAQ,eAAe,QAAQ,SAAS,QAAQ,CAAC,WAAW,EAAE,WAAW,MAAM,gBAAgB,mBAAmB,EAAE,CAAA;IACjI,IAAI,kBAAkB,GAAG,4BAA4B,EAAE,CAAC;QACtD,GAAG,IAAI,cAAc,kBAAkB,EAAE,CAAA;IAC3C,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,IAAI,YAAY,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC;QACpC,MAAM;QACN,YAAY,EAAE,aAAa;QAC3B,GAAG;QACH,SAAS;QACT,8BAA8B,EAAE,IAAI;QACpC,cAAc,EAAE,mBAAmB,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY;KACvE,CAAC,CAAA;IACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAA;IACxE,MAAM,kBAAkB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,wBAAwB,CAAC,IAAI,4BAA4B,CAAC,CAAA;IAEnH,KAAK,CAAC,iEAAiE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAA;IACrG,MAAM,OAAO,GAAkB;QAC7B,QAAQ;QACR,SAAS,EAAE,cAAc,CAAC,YAAY;QACtC,SAAS,EAAE,iBAAiB;KAC7B,CAAA;IACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;IAEhC,OAAO;QACL,SAAS,EAAE,iBAAiB;QAC5B,kBAAkB;KACnB,CAAA;AACH,CAAC"}
1
+ {"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,OAAO,CAAA;AACvB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAGlH,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,CAAA;AAE/B,MAAM,4BAA4B,GAAG,CAAC,CAAA;AAEtC,IAAI,YAAY,EAAE,CAAC;IACjB,KAAK,CAAC,mDAAmD,CAAC,CAAA;AAC5D,CAAC;KAAM,CAAC;IACN,UAAW,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,MAAoB,EAAE,EAAE;QACjD,IAAI,MAAM,2DAAkC,EAAE,CAAC;YAC7C,cAAc,EAAE,CAAA;YAChB,UAAW,CAAC,WAAW,4DAAiC,CAAA;QAC1D,CAAC;IACH,CAAC,CAAC,CAAA;IACF,iBAAiB,CAAC,UAA8B,CAAC,CAAA;AACnD,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;IAChD,KAAK,CAAC,wCAAwC,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;IAC7D,MAAM,GAAG,CAAA;AACX,CAAC,CAAC,CAAA;AAEF,KAAK,UAAU,iBAAiB,CAAC,OAAyB;IACxD,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,CAAA;IACxC,MAAM,uCAAuC,GAAG,CAAC,CAAA;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAA;IAC5B,mEAAmE;IACnE,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEhD,2HAA2H;IAC3H,qEAAqE;IACrE,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,sBAAsB,CAAC,CAAA;IAExH,gEAAgE;IAChE,MAAM,QAAQ,GAAG,GAAG,OAAO,CAAC,QAAQ,UAAU,OAAO,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;IAEnF,MAAM,iBAAiB,GACrB,OAAO,OAAO,CAAC,2BAA2B,KAAK,QAAQ;QACrD,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,2BAA2B,EAAE,uCAAuC,CAAC;QACxF,CAAC,CAAC,EAAE,CAAA;IACR,MAAM,YAAY,GAAG,iBAAiB,GAAG,sBAAsB,CAAA;IAE/D,MAAM,0CAA0C,GAAG,MAAM,wBAAwB,CAC/E,OAAO,EACP,mBAAmB,EACnB,YAAY,EACZ,sBAAsB,CACvB,CAAA;IAED,MAAM,0BAA0B,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,0CAA0C,EAAE,iBAAiB,CAAC,CAAA;IAE3H,2DAA2D;IAC3D,KAAK,IAAI,MAAM,GAAG,0CAA0C,EAAE,MAAM,GAAG,mBAAmB,EAAE,MAAM,EAAE,EAAE,CAAC;QACrG,MAAM,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,GAAG,sBAAsB,CAAA;QACrF,MAAM,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;QAEzE,IAAI,gBAAgB,GAAG,6BAA6B,EAAE,CAAC;YACrD,MAAM,QAAQ,GAAG,gBAAgB,GAAG,6BAA6B,GAAG,GAAG,CAAA;YAEvE,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtB,CAAC;QACD,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAA;IAC5D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,OAAyB,EACzB,mBAA2B,EAC3B,YAAoB,EACpB,oBAA4B;IAE5B,MAAM,0BAA0B,GAAG,OAAO,CAAC,2BAA2B,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACxI,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAChC,OAAO,mBAAmB,CAAA;IAC5B,CAAC;IAED,IAAI,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IAEvE,4EAA4E;IAC5E,IAAI,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/D,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,6BAA6B,CAAA;QAClF,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,CAAA;QAC7B,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,6BAA6B,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,YAAY,CAAA;IACnE,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,6BAA6B,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,GAAG,oBAAoB,CAAC,CAAA;IAC7H,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAA;AAC1E,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,OAAyB,EACzB,OAAiB,EACjB,QAAgB,EAChB,mBAA2B,EAC3B,gBAAwB;IAExB,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAM;IACR,CAAC;IAED,iHAAiH;IACjH,qEAAqE;IACrE,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAElI,kGAAkG;IAClG,MAAM,UAAU,GACd,mBAAmB,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAA;IAEpI,MAAM,eAAe,GACnB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,kBAAkB,EAAE,SAAS,CAAC,kBAAkB,CAAC,CAAA;IAC7H,MAAM,YAAY,GAAa,EAAE,CAAA;IACjC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,mBAAmB,GAAG,CAAC,EAAE,MAAM,IAAI,eAAe,EAAE,CAAC;QACjF,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC3B,CAAC;IAED,wHAAwH;IACxH,MAAM,iBAAiB,CAAC,YAAY,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACvE,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,mBAAmB,GAAG,CAAC,GAAG,MAAM,CAAC,CAAA;QACtF,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;IAChF,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,SAAS,EAAoB,EAC1F,MAAc,EACd,OAAiB,EACjB,QAAgB,EAChB,kBAAkB,GAAG,4BAA4B,EACjD,QAAQ,GAAG,IAAI;IAEf,MAAM,cAAc,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnD,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,CAAA;IAC7C,MAAM,eAAe,GAAG,kBAAkB,KAAK,4BAA4B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,kBAAkB,EAAE,CAAA;IAChH,MAAM,aAAa,GAAG,GAAG,QAAQ,IAAI,gBAAgB,CAAC,cAAc,CAAC,GAAG,eAAe,OAAO,CAAA;IAC9F,MAAM,aAAa,GAAG,GAAG,aAAa,MAAM,CAAA;IAC5C,MAAM,aAAa,GAAG,GAAG,aAAa,KAAK,CAAA;IAC3C,IAAI,eAAe,CAAA;IACnB,IAAI,QAAQ,EAAE,CAAC;QACb,eAAe,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAA;IACrH,CAAC;IAED,IAAI,eAAe,KAAK,SAAS,EAAE,CAAC;QAClC,KAAK,CAAC,oDAAoD,EAAE,QAAQ,EAAE,kBAAkB,CAAC,CAAA;QACzF,MAAM,OAAO,GAAkB;YAC7B,QAAQ;YACR,SAAS,EAAE,eAAe;YAC1B,SAAS,EAAE,kBAAkB;SAC9B,CAAA;QACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QAChC,OAAO;YACL,SAAS,EAAE,kBAAkB;YAC7B,kBAAkB,EAAE,4BAA4B;SACjD,CAAA;IACH,CAAC;IAED,IAAI,GAAG,GAAG,GAAG,QAAQ,eAAe,QAAQ,SAAS,QAAQ,CAAC,WAAW,EAAE,WAAW,MAAM,gBAAgB,mBAAmB,EAAE,CAAA;IACjI,IAAI,kBAAkB,GAAG,4BAA4B,EAAE,CAAC;QACtD,GAAG,IAAI,cAAc,kBAAkB,EAAE,CAAA;IAC3C,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,IAAI,YAAY,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC;QACpC,MAAM;QACN,YAAY,EAAE,aAAa;QAC3B,GAAG;QACH,SAAS;QACT,8BAA8B,EAAE,IAAI;QACpC,cAAc,EAAE,mBAAmB,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY;KACvE,CAAC,CAAA;IACF,MAAM,iBAAiB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAA;IACxE,MAAM,kBAAkB,GAAG,MAAM,CAAC,cAAc,CAAC,OAAO,CAAC,wBAAwB,CAAC,IAAI,4BAA4B,CAAC,CAAA;IAEnH,KAAK,CAAC,iEAAiE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAA;IACrG,MAAM,OAAO,GAAkB;QAC7B,QAAQ;QACR,SAAS,EAAE,cAAc,CAAC,YAAY;QACtC,SAAS,EAAE,iBAAiB;KAC7B,CAAA;IACD,UAAW,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;IAEhC,OAAO;QACL,SAAS,EAAE,iBAAiB;QAC5B,kBAAkB;KACnB,CAAA;AACH,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tardis-dev",
3
- "version": "17.3.0",
3
+ "version": "18.0.1",
4
4
  "engines": {
5
5
  "node": ">=24.5"
6
6
  },
@@ -70,7 +70,6 @@
70
70
  "dependencies": {
71
71
  "debug": "^4.4.3",
72
72
  "follow-redirects": "^1.16.0",
73
- "p-map": "^7.0.5",
74
73
  "protobufjs": "^8.7.1",
75
74
  "ws": "^8.21.1"
76
75
  },
@@ -0,0 +1,118 @@
1
+ type PendingRead<T> = {
2
+ resolve: (result: IteratorResult<T>) => void
3
+ reject: (error: Error) => void
4
+ }
5
+
6
+ type PendingWrite<T> = {
7
+ value: T
8
+ resolve: (written: boolean) => void
9
+ }
10
+
11
+ export class BoundedQueue<T> {
12
+ private readonly values: (T | undefined)[]
13
+ private readIndex = 0
14
+ private size = 0
15
+ private pendingWrites: PendingWrite<T>[] = []
16
+ private pendingRead?: PendingRead<T>
17
+ private failure?: Error
18
+ private open = true
19
+
20
+ constructor(private readonly capacity: number) {
21
+ this.values = new Array(capacity)
22
+ }
23
+
24
+ get isOpen() {
25
+ return this.open
26
+ }
27
+
28
+ write(value: T): boolean | Promise<boolean> {
29
+ if (!this.open) {
30
+ return false
31
+ }
32
+
33
+ if (this.pendingRead !== undefined) {
34
+ const { resolve } = this.pendingRead
35
+ this.pendingRead = undefined
36
+ resolve({ done: false, value })
37
+ return true
38
+ }
39
+
40
+ if (this.size < this.capacity) {
41
+ this.values[(this.readIndex + this.size) % this.capacity] = value
42
+ this.size++
43
+ return true
44
+ }
45
+
46
+ return new Promise((resolve) => this.pendingWrites.push({ value, resolve }))
47
+ }
48
+
49
+ async next(): Promise<IteratorResult<T>> {
50
+ if (this.size > 0) {
51
+ const value = this.values[this.readIndex] as T
52
+ this.values[this.readIndex] = undefined
53
+ this.readIndex = (this.readIndex + 1) % this.capacity
54
+ this.size--
55
+ const pendingWrite = this.pendingWrites.shift()
56
+ if (pendingWrite !== undefined) {
57
+ this.values[(this.readIndex + this.size) % this.capacity] = pendingWrite.value
58
+ this.size++
59
+ pendingWrite.resolve(true)
60
+ }
61
+ return { done: false, value }
62
+ }
63
+
64
+ if (this.failure !== undefined) {
65
+ throw this.failure
66
+ }
67
+
68
+ if (!this.open) {
69
+ return { done: true, value: undefined }
70
+ }
71
+
72
+ return new Promise((resolve, reject) => {
73
+ this.pendingRead = { resolve, reject }
74
+ })
75
+ }
76
+
77
+ finish() {
78
+ if (!this.open) {
79
+ return
80
+ }
81
+
82
+ this.open = false
83
+ this.pendingRead?.resolve({ done: true, value: undefined })
84
+ this.pendingRead = undefined
85
+ }
86
+
87
+ fail(error: Error) {
88
+ if (!this.open) {
89
+ return
90
+ }
91
+
92
+ this.failure = error
93
+ this.stop(error)
94
+ }
95
+
96
+ close() {
97
+ this.stop()
98
+ }
99
+
100
+ private stop(error?: Error) {
101
+ this.open = false
102
+ this.values.fill(undefined)
103
+ this.readIndex = 0
104
+ this.size = 0
105
+ for (const { resolve } of this.pendingWrites) {
106
+ resolve(false)
107
+ }
108
+ this.pendingWrites = []
109
+ if (this.pendingRead !== undefined) {
110
+ if (error === undefined) {
111
+ this.pendingRead.resolve({ done: true, value: undefined })
112
+ } else {
113
+ this.pendingRead.reject(error)
114
+ }
115
+ this.pendingRead = undefined
116
+ }
117
+ }
118
+ }
package/src/combine.ts CHANGED
@@ -1,5 +1,4 @@
1
- import { PassThrough } from 'stream'
2
- import { once } from 'events'
1
+ import { closeIterator, isManagedRealTimeIterator, mergeRealTime } from './realtimeiterator.ts'
3
2
 
4
3
  type NextMessageResultWithIndex = {
5
4
  index: number
@@ -11,6 +10,12 @@ type Combinable = { localTimestamp: Date }
11
10
  const DATE_MAX = new Date(8640000000000000)
12
11
 
13
12
  type OffsetMS = number | ((message: Combinable) => number)
13
+ type IteratorPayload = AsyncIterableIterator<Combinable>[] | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS }[]
14
+ type CombinedMessage<T> = T extends AsyncIterableIterator<infer U>[]
15
+ ? U
16
+ : T extends { stream: AsyncIterableIterator<infer U> }[]
17
+ ? U
18
+ : never
14
19
 
15
20
  async function nextWithIndex(
16
21
  iterator: AsyncIterableIterator<Combinable> | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS },
@@ -71,56 +76,27 @@ function findOldestResult(oldest: NextMessageResultWithIndex, current: NextMessa
71
76
 
72
77
  // combines multiple iterators from for example multiple exchanges
73
78
  // works both for real-time and historical data
74
- export async function* combine<
75
- T extends AsyncIterableIterator<Combinable>[] | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS }[]
76
- >(
77
- ...iteratorsPayload: T
78
- ): AsyncIterableIterator<
79
- T extends AsyncIterableIterator<infer U>[] ? U : T extends { stream: AsyncIterableIterator<infer Z> }[] ? Z : never
80
- > {
79
+ export function combine<T extends IteratorPayload>(...iteratorsPayload: T): AsyncIterableIterator<CombinedMessage<T>> {
81
80
  const iterators = iteratorsPayload.map((payload) => {
82
81
  if ('stream' in payload) {
83
82
  return payload.stream
84
83
  }
85
84
  return payload
86
85
  })
87
-
88
- if (iterators.length === 0) {
89
- return
86
+ if (isManagedRealTimeIterator(iterators[0])) {
87
+ return mergeRealTime(iterators) as AsyncIterableIterator<CombinedMessage<T>>
90
88
  }
91
- // decide based on first provided iterator if we're dealing with real-time or historical data streams
92
- if ((iterators[0] as any).__realtime__) {
93
- const combinedStream = new PassThrough({
94
- objectMode: true,
95
- highWaterMark: 8096
96
- })
97
-
98
- combinedStream.setMaxListeners(iterators.length + 1)
99
-
100
- iterators.forEach(async function writeMessagesToCombinedStream(messages) {
101
- for await (const message of messages) {
102
- if (combinedStream.destroyed) {
103
- return
104
- }
105
89
 
106
- if (!combinedStream.write(message)) {
107
- // Handle backpressure on write
108
- await once(combinedStream, 'drain')
109
- }
110
- }
111
- })
112
-
113
- for await (const message of combinedStream) {
114
- yield message
115
- }
116
- } else {
117
- return yield* combineHistorical(iteratorsPayload) as any
118
- }
90
+ return combineHistorical(iteratorsPayload) as AsyncIterableIterator<CombinedMessage<T>>
119
91
  }
120
92
 
121
93
  async function* combineHistorical(
122
94
  iterators: AsyncIterableIterator<Combinable>[] | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS }[]
123
95
  ) {
96
+ if (iterators.length === 0) {
97
+ return
98
+ }
99
+
124
100
  try {
125
101
  // wait for all results to resolve
126
102
  const results = await Promise.all(iterators.map(nextWithIndex))
@@ -153,8 +129,6 @@ async function* combineHistorical(
153
129
  }
154
130
  } while (aliveIteratorsCount > 0)
155
131
  } finally {
156
- for (let iterator of iterators) {
157
- ;(iterator as any).return()
158
- }
132
+ await Promise.all(iterators.map((iterator) => closeIterator('stream' in iterator ? iterator.stream : iterator)))
159
133
  }
160
134
  }
@@ -1,4 +1,5 @@
1
1
  import { Disconnect, Exchange, NormalizedData } from '../types.ts'
2
+ import { closeIterator, createManagedRealTimeIterator, isManagedRealTimeIterator } from '../realtimeiterator.ts'
2
3
  import { ComputableContext, createWithContext } from './context.ts'
3
4
 
4
5
  export type Computable<T extends NormalizedData> = {
@@ -39,7 +40,7 @@ async function* _compute<T extends ComputableFactory<any>[], U extends Normalize
39
40
  }
40
41
  }
41
42
  } finally {
42
- messages.return!()
43
+ await closeIterator(messages)
43
44
  }
44
45
  }
45
46
 
@@ -47,13 +48,13 @@ export function compute<T extends ComputableFactory<any>[], U extends Normalized
47
48
  messages: AsyncIterableIterator<U>,
48
49
  ...computables: T
49
50
  ): AsyncIterableIterator<T extends ComputableFactory<infer Z>[] ? (U extends Disconnect ? U | Z | Disconnect : U | Z) : never> {
50
- let _iterator = _compute(messages, ...computables)
51
+ const iterator = _compute(messages, ...computables)
51
52
 
52
- if ((messages as any).__realtime__ === true) {
53
- ;(_iterator as any).__realtime__ = true
53
+ if (isManagedRealTimeIterator(messages)) {
54
+ return createManagedRealTimeIterator(iterator, () => closeIterator(messages))
54
55
  }
55
56
 
56
- return _iterator
57
+ return iterator
57
58
  }
58
59
 
59
60
  class Computables {
@@ -1,7 +1,7 @@
1
1
  import { existsSync } from 'node:fs'
2
- import pMap from 'p-map'
3
2
  import { debug } from './debug.ts'
4
3
  import { DatasetType } from './exchangedetails.ts'
4
+ import { forEachConcurrent } from './foreachconcurrent.ts'
5
5
  import { addDays, doubleDigit, download, parseAsUTCDate, sequence } from './handy.ts'
6
6
  import { getOptions } from './options.ts'
7
7
  import { Exchange } from './types.ts'
@@ -62,8 +62,9 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
62
62
  )
63
63
 
64
64
  // download the rest concurrently up to the CONCURRENCY_LIMIT
65
- await pMap(
65
+ await forEachConcurrent(
66
66
  sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
67
+ CONCURRENCY_LIMIT,
67
68
  (offset) =>
68
69
  downloadDataSet(
69
70
  getDownloadOptions({
@@ -77,8 +78,7 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
77
78
  date: addDays(startDate, offset)
78
79
  }),
79
80
  skipIfExists
80
- ),
81
- { concurrency: CONCURRENCY_LIMIT }
81
+ )
82
82
  )
83
83
  const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000
84
84
 
@@ -0,0 +1,29 @@
1
+ export async function forEachConcurrent<T>(items: Iterable<T>, concurrency: number, action: (item: T) => Promise<unknown>) {
2
+ const iterator = items[Symbol.iterator]()
3
+ let hasError = false
4
+ let firstError: unknown
5
+
6
+ const runWorker = async () => {
7
+ while (hasError === false) {
8
+ const next = iterator.next()
9
+ if (next.done) {
10
+ return
11
+ }
12
+
13
+ try {
14
+ await action(next.value)
15
+ } catch (error) {
16
+ if (hasError === false) {
17
+ hasError = true
18
+ firstError = error
19
+ }
20
+ return
21
+ }
22
+ }
23
+ }
24
+
25
+ await Promise.all(Array.from({ length: concurrency }, () => runWorker()))
26
+ if (hasError) {
27
+ throw firstError
28
+ }
29
+ }