tardis-machine 18.0.0 → 18.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/benchmark.js +35 -14
  2. package/bin/tardis-machine.js +0 -1
  3. package/dist/tardismachine.d.ts.map +1 -1
  4. package/dist/tardismachine.js +20 -55
  5. package/dist/tardismachine.js.map +1 -1
  6. package/dist/ws/replay.d.ts +2 -2
  7. package/dist/ws/replay.d.ts.map +1 -1
  8. package/dist/ws/replay.js +11 -5
  9. package/dist/ws/replay.js.map +1 -1
  10. package/dist/ws/replaynormalized.d.ts +2 -2
  11. package/dist/ws/replaynormalized.d.ts.map +1 -1
  12. package/dist/ws/replaynormalized.js +7 -5
  13. package/dist/ws/replaynormalized.js.map +1 -1
  14. package/dist/ws/server.d.ts +20 -0
  15. package/dist/ws/server.d.ts.map +1 -0
  16. package/dist/ws/server.js +154 -0
  17. package/dist/ws/server.js.map +1 -0
  18. package/dist/ws/streamnormalized.d.ts +2 -2
  19. package/dist/ws/streamnormalized.d.ts.map +1 -1
  20. package/dist/ws/streamnormalized.js +7 -5
  21. package/dist/ws/streamnormalized.js.map +1 -1
  22. package/dist/ws/subscriptionsmappers.d.ts +1 -1
  23. package/dist/ws/subscriptionsmappers.d.ts.map +1 -1
  24. package/node_modules/tardis-dev/README.md +1 -1
  25. package/node_modules/tardis-dev/dist/debug.d.ts +12 -2
  26. package/node_modules/tardis-dev/dist/debug.d.ts.map +1 -1
  27. package/node_modules/tardis-dev/dist/debug.js +2 -1
  28. package/node_modules/tardis-dev/dist/debug.js.map +1 -1
  29. package/node_modules/tardis-dev/dist/downloaddatasets.d.ts.map +1 -1
  30. package/node_modules/tardis-dev/dist/downloaddatasets.js +3 -1
  31. package/node_modules/tardis-dev/dist/downloaddatasets.js.map +1 -1
  32. package/node_modules/tardis-dev/dist/handy.d.ts +5 -2
  33. package/node_modules/tardis-dev/dist/handy.d.ts.map +1 -1
  34. package/node_modules/tardis-dev/dist/handy.js +77 -90
  35. package/node_modules/tardis-dev/dist/handy.js.map +1 -1
  36. package/node_modules/tardis-dev/dist/mappers/hyperliquid.d.ts +1 -1
  37. package/node_modules/tardis-dev/dist/mappers/hyperliquid.d.ts.map +1 -1
  38. package/node_modules/tardis-dev/dist/mappers/hyperliquid.js +15 -4
  39. package/node_modules/tardis-dev/dist/mappers/hyperliquid.js.map +1 -1
  40. package/node_modules/tardis-dev/dist/mappers/index.d.ts +1 -1
  41. package/node_modules/tardis-dev/dist/realtimefeeds/mexc.d.ts +0 -7
  42. package/node_modules/tardis-dev/dist/realtimefeeds/mexc.d.ts.map +1 -1
  43. package/node_modules/tardis-dev/dist/realtimefeeds/mexc.js +2 -70
  44. package/node_modules/tardis-dev/dist/realtimefeeds/mexc.js.map +1 -1
  45. package/node_modules/tardis-dev/dist/realtimefeeds/mexcprotobuf.d.ts +43 -0
  46. package/node_modules/tardis-dev/dist/realtimefeeds/mexcprotobuf.d.ts.map +1 -0
  47. package/node_modules/tardis-dev/dist/realtimefeeds/mexcprotobuf.js +294 -0
  48. package/node_modules/tardis-dev/dist/realtimefeeds/mexcprotobuf.js.map +1 -0
  49. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.d.ts +3 -3
  50. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.d.ts.map +1 -1
  51. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.js +3 -3
  52. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.js.map +1 -1
  53. package/node_modules/tardis-dev/dist/replay.d.ts.map +1 -1
  54. package/node_modules/tardis-dev/dist/replay.js +6 -4
  55. package/node_modules/tardis-dev/dist/replay.js.map +1 -1
  56. package/node_modules/tardis-dev/dist/worker.d.ts.map +1 -1
  57. package/node_modules/tardis-dev/dist/worker.js +7 -4
  58. package/node_modules/tardis-dev/dist/worker.js.map +1 -1
  59. package/node_modules/tardis-dev/dist/zstd.d.ts +3 -0
  60. package/node_modules/tardis-dev/dist/zstd.d.ts.map +1 -0
  61. package/node_modules/tardis-dev/dist/zstd.js +122 -0
  62. package/node_modules/tardis-dev/dist/zstd.js.map +1 -0
  63. package/node_modules/tardis-dev/package.json +3 -7
  64. package/node_modules/tardis-dev/src/debug.ts +16 -1
  65. package/node_modules/tardis-dev/src/downloaddatasets.ts +3 -1
  66. package/node_modules/tardis-dev/src/external-modules.d.ts +5 -0
  67. package/node_modules/tardis-dev/src/handy.ts +100 -93
  68. package/node_modules/tardis-dev/src/mappers/hyperliquid.ts +19 -5
  69. package/node_modules/tardis-dev/src/realtimefeeds/mexc.ts +2 -72
  70. package/node_modules/tardis-dev/src/realtimefeeds/mexcprotobuf.ts +364 -0
  71. package/node_modules/tardis-dev/src/realtimefeeds/realtimefeed.ts +5 -5
  72. package/node_modules/tardis-dev/src/replay.ts +6 -4
  73. package/node_modules/tardis-dev/src/worker.ts +7 -4
  74. package/node_modules/tardis-dev/src/zstd.ts +139 -0
  75. package/package.json +9 -35
  76. package/src/tardismachine.ts +25 -66
  77. package/src/ws/replay.ts +13 -8
  78. package/src/ws/replaynormalized.ts +8 -6
  79. package/src/ws/server.ts +173 -0
  80. package/src/ws/streamnormalized.ts +8 -6
  81. package/src/ws/subscriptionsmappers.ts +1 -1
  82. package/node_modules/follow-redirects/LICENSE +0 -18
  83. package/node_modules/follow-redirects/README.md +0 -157
  84. package/node_modules/follow-redirects/debug.js +0 -15
  85. package/node_modules/follow-redirects/http.js +0 -1
  86. package/node_modules/follow-redirects/https.js +0 -1
  87. package/node_modules/follow-redirects/index.js +0 -709
  88. package/node_modules/follow-redirects/package.json +0 -58
  89. package/node_modules/long/LICENSE +0 -202
  90. package/node_modules/long/README.md +0 -286
  91. package/node_modules/long/index.d.ts +0 -2
  92. package/node_modules/long/index.js +0 -1581
  93. package/node_modules/long/package.json +0 -58
  94. package/node_modules/long/types.d.ts +0 -474
  95. package/node_modules/long/umd/index.d.ts +0 -3
  96. package/node_modules/long/umd/index.js +0 -1622
  97. package/node_modules/long/umd/package.json +0 -3
  98. package/node_modules/long/umd/types.d.ts +0 -474
  99. package/node_modules/protobufjs/LICENSE +0 -39
  100. package/node_modules/protobufjs/README.md +0 -440
  101. package/node_modules/protobufjs/dist/light/protobuf.js +0 -9276
  102. package/node_modules/protobufjs/dist/light/protobuf.js.map +0 -1
  103. package/node_modules/protobufjs/dist/light/protobuf.min.js +0 -8
  104. package/node_modules/protobufjs/dist/light/protobuf.min.js.map +0 -1
  105. package/node_modules/protobufjs/dist/minimal/protobuf.js +0 -3556
  106. package/node_modules/protobufjs/dist/minimal/protobuf.js.map +0 -1
  107. package/node_modules/protobufjs/dist/minimal/protobuf.min.js +0 -8
  108. package/node_modules/protobufjs/dist/minimal/protobuf.min.js.map +0 -1
  109. package/node_modules/protobufjs/dist/protobuf.js +0 -11233
  110. package/node_modules/protobufjs/dist/protobuf.js.map +0 -1
  111. package/node_modules/protobufjs/dist/protobuf.min.js +0 -8
  112. package/node_modules/protobufjs/dist/protobuf.min.js.map +0 -1
  113. package/node_modules/protobufjs/ext/README.md +0 -70
  114. package/node_modules/protobufjs/ext/debug/README.md +0 -4
  115. package/node_modules/protobufjs/ext/debug/index.js +0 -71
  116. package/node_modules/protobufjs/ext/descriptor/README.md +0 -5
  117. package/node_modules/protobufjs/ext/descriptor/index.d.ts +0 -2
  118. package/node_modules/protobufjs/ext/descriptor/index.js +0 -2
  119. package/node_modules/protobufjs/ext/descriptor.d.ts +0 -87
  120. package/node_modules/protobufjs/ext/descriptor.generated.d.ts +0 -417
  121. package/node_modules/protobufjs/ext/descriptor.js +0 -1356
  122. package/node_modules/protobufjs/ext/protojson.LICENSE +0 -201
  123. package/node_modules/protobufjs/ext/protojson.d.ts +0 -20
  124. package/node_modules/protobufjs/ext/protojson.generated.d.ts +0 -49
  125. package/node_modules/protobufjs/ext/protojson.js +0 -951
  126. package/node_modules/protobufjs/ext/textformat.d.ts +0 -19
  127. package/node_modules/protobufjs/ext/textformat.generated.d.ts +0 -33
  128. package/node_modules/protobufjs/ext/textformat.js +0 -1276
  129. package/node_modules/protobufjs/google/LICENSE +0 -27
  130. package/node_modules/protobufjs/google/README.md +0 -1
  131. package/node_modules/protobufjs/google/api/annotations.json +0 -85
  132. package/node_modules/protobufjs/google/api/annotations.proto +0 -11
  133. package/node_modules/protobufjs/google/api/http.json +0 -87
  134. package/node_modules/protobufjs/google/api/http.proto +0 -31
  135. package/node_modules/protobufjs/google/protobuf/api.json +0 -124
  136. package/node_modules/protobufjs/google/protobuf/api.proto +0 -34
  137. package/node_modules/protobufjs/google/protobuf/compiler/plugin.json +0 -126
  138. package/node_modules/protobufjs/google/protobuf/compiler/plugin.proto +0 -47
  139. package/node_modules/protobufjs/google/protobuf/descriptor.json +0 -1465
  140. package/node_modules/protobufjs/google/protobuf/descriptor.proto +0 -527
  141. package/node_modules/protobufjs/google/protobuf/source_context.json +0 -21
  142. package/node_modules/protobufjs/google/protobuf/source_context.proto +0 -7
  143. package/node_modules/protobufjs/google/protobuf/type.json +0 -209
  144. package/node_modules/protobufjs/google/protobuf/type.proto +0 -89
  145. package/node_modules/protobufjs/index.d.ts +0 -3203
  146. package/node_modules/protobufjs/index.js +0 -4
  147. package/node_modules/protobufjs/light.d.ts +0 -2
  148. package/node_modules/protobufjs/light.js +0 -4
  149. package/node_modules/protobufjs/minimal.d.ts +0 -2
  150. package/node_modules/protobufjs/minimal.js +0 -4
  151. package/node_modules/protobufjs/package.json +0 -93
  152. package/node_modules/protobufjs/src/common.js +0 -405
  153. package/node_modules/protobufjs/src/converter.js +0 -352
  154. package/node_modules/protobufjs/src/decoder.js +0 -279
  155. package/node_modules/protobufjs/src/encoder.js +0 -123
  156. package/node_modules/protobufjs/src/enum.js +0 -239
  157. package/node_modules/protobufjs/src/field.js +0 -505
  158. package/node_modules/protobufjs/src/index-light.js +0 -104
  159. package/node_modules/protobufjs/src/index-minimal.js +0 -35
  160. package/node_modules/protobufjs/src/index.js +0 -12
  161. package/node_modules/protobufjs/src/mapfield.js +0 -144
  162. package/node_modules/protobufjs/src/message.js +0 -137
  163. package/node_modules/protobufjs/src/method.js +0 -183
  164. package/node_modules/protobufjs/src/namespace.js +0 -574
  165. package/node_modules/protobufjs/src/object.js +0 -392
  166. package/node_modules/protobufjs/src/oneof.js +0 -233
  167. package/node_modules/protobufjs/src/parse.js +0 -1111
  168. package/node_modules/protobufjs/src/reader.js +0 -788
  169. package/node_modules/protobufjs/src/reader_buffer.js +0 -77
  170. package/node_modules/protobufjs/src/root.js +0 -424
  171. package/node_modules/protobufjs/src/roots.js +0 -18
  172. package/node_modules/protobufjs/src/rpc/service.js +0 -155
  173. package/node_modules/protobufjs/src/rpc.js +0 -36
  174. package/node_modules/protobufjs/src/service.js +0 -206
  175. package/node_modules/protobufjs/src/tokenize.js +0 -421
  176. package/node_modules/protobufjs/src/type.js +0 -676
  177. package/node_modules/protobufjs/src/types.js +0 -196
  178. package/node_modules/protobufjs/src/typescript.js +0 -25
  179. package/node_modules/protobufjs/src/util/aspromise.d.ts +0 -13
  180. package/node_modules/protobufjs/src/util/aspromise.js +0 -52
  181. package/node_modules/protobufjs/src/util/base64.d.ts +0 -32
  182. package/node_modules/protobufjs/src/util/base64.js +0 -146
  183. package/node_modules/protobufjs/src/util/codegen.d.ts +0 -31
  184. package/node_modules/protobufjs/src/util/codegen.js +0 -118
  185. package/node_modules/protobufjs/src/util/eventemitter.d.ts +0 -45
  186. package/node_modules/protobufjs/src/util/eventemitter.js +0 -86
  187. package/node_modules/protobufjs/src/util/fetch.d.ts +0 -56
  188. package/node_modules/protobufjs/src/util/fetch.js +0 -112
  189. package/node_modules/protobufjs/src/util/float.d.ts +0 -83
  190. package/node_modules/protobufjs/src/util/float.js +0 -335
  191. package/node_modules/protobufjs/src/util/fs.js +0 -11
  192. package/node_modules/protobufjs/src/util/longbits.js +0 -204
  193. package/node_modules/protobufjs/src/util/minimal.js +0 -511
  194. package/node_modules/protobufjs/src/util/path.d.ts +0 -22
  195. package/node_modules/protobufjs/src/util/path.js +0 -100
  196. package/node_modules/protobufjs/src/util/patterns.js +0 -7
  197. package/node_modules/protobufjs/src/util/pool.d.ts +0 -32
  198. package/node_modules/protobufjs/src/util/pool.js +0 -48
  199. package/node_modules/protobufjs/src/util/utf8.d.ts +0 -24
  200. package/node_modules/protobufjs/src/util/utf8.js +0 -190
  201. package/node_modules/protobufjs/src/util.js +0 -264
  202. package/node_modules/protobufjs/src/verifier.js +0 -185
  203. package/node_modules/protobufjs/src/wrappers.js +0 -106
  204. package/node_modules/protobufjs/src/writer.js +0 -700
  205. package/node_modules/protobufjs/src/writer_buffer.js +0 -100
  206. package/node_modules/protobufjs/tsconfig.json +0 -6
  207. package/node_modules/uWebSockets.js/ESM_wrapper.mjs +0 -53
  208. package/node_modules/uWebSockets.js/LICENSE +0 -201
  209. package/node_modules/uWebSockets.js/README.md +0 -6
  210. package/node_modules/uWebSockets.js/index.d.ts +0 -468
  211. package/node_modules/uWebSockets.js/package.json +0 -12
  212. package/node_modules/uWebSockets.js/source_commit +0 -1
  213. package/node_modules/uWebSockets.js/uws.js +0 -76
  214. package/node_modules/uWebSockets.js/uws_darwin_arm64_127.node +0 -0
  215. package/node_modules/uWebSockets.js/uws_darwin_arm64_137.node +0 -0
  216. package/node_modules/uWebSockets.js/uws_darwin_arm64_147.node +0 -0
  217. package/node_modules/uWebSockets.js/uws_darwin_x64_127.node +0 -0
  218. package/node_modules/uWebSockets.js/uws_darwin_x64_137.node +0 -0
  219. package/node_modules/uWebSockets.js/uws_darwin_x64_147.node +0 -0
  220. package/node_modules/uWebSockets.js/uws_linux_arm64_127.node +0 -0
  221. package/node_modules/uWebSockets.js/uws_linux_arm64_137.node +0 -0
  222. package/node_modules/uWebSockets.js/uws_linux_arm64_147.node +0 -0
  223. package/node_modules/uWebSockets.js/uws_linux_x64_127.node +0 -0
  224. package/node_modules/uWebSockets.js/uws_linux_x64_137.node +0 -0
  225. package/node_modules/uWebSockets.js/uws_linux_x64_147.node +0 -0
  226. package/node_modules/uWebSockets.js/uws_win32_x64_127.node +0 -0
  227. package/node_modules/uWebSockets.js/uws_win32_x64_137.node +0 -0
  228. package/node_modules/uWebSockets.js/uws_win32_x64_147.node +0 -0
@@ -0,0 +1,364 @@
1
+ const textDecoder = new TextDecoder()
2
+
3
+ // Decodes only the current MEXC real-time channels supported by MexcRealTimeFeed.
4
+ // Field numbers mirror the official schemas at commit 7b8ac7a6681f28551612a5a7cefbb7e09b56bb85:
5
+ // https://github.com/mexcdevelop/websocket-proto/tree/7b8ac7a6681f28551612a5a7cefbb7e09b56bb85
6
+ // int64 values remain decimal strings and repeated fields remain empty arrays when absent,
7
+ // preserving the output shape previously produced by protobufjs.
8
+ export function decodeMexcProtobufMessage(data: Uint8Array) {
9
+ const reader = new ProtobufReader(data)
10
+ const message: {
11
+ channel?: string
12
+ symbol?: string
13
+ symbolId?: string
14
+ createTime?: string
15
+ sendTime?: string
16
+ publicAggreDepths?: ReturnType<typeof decodeDepths>
17
+ publicAggreDeals?: ReturnType<typeof decodeDeals>
18
+ publicAggreBookTicker?: ReturnType<typeof decodeBookTicker>
19
+ } = {}
20
+
21
+ while (reader.done === false) {
22
+ const { fieldNumber, wireType } = reader.readTag()
23
+
24
+ switch (fieldNumber) {
25
+ case 1:
26
+ message.channel = reader.readString(wireType)
27
+ break
28
+ case 3:
29
+ message.symbol = reader.readString(wireType)
30
+ break
31
+ case 4:
32
+ message.symbolId = reader.readString(wireType)
33
+ break
34
+ case 5:
35
+ message.createTime = reader.readInt64String(wireType)
36
+ break
37
+ case 6:
38
+ message.sendTime = reader.readInt64String(wireType)
39
+ break
40
+ case 313:
41
+ message.publicAggreDepths = decodeDepths(reader.readBytes(wireType))
42
+ break
43
+ case 314:
44
+ message.publicAggreDeals = decodeDeals(reader.readBytes(wireType))
45
+ break
46
+ case 315:
47
+ message.publicAggreBookTicker = decodeBookTicker(reader.readBytes(wireType))
48
+ break
49
+ default:
50
+ reader.skipField(wireType)
51
+ }
52
+ }
53
+
54
+ return message
55
+ }
56
+
57
+ function decodeDeals(data: Uint8Array) {
58
+ const reader = new ProtobufReader(data)
59
+ const message: { deals: ReturnType<typeof decodeTrade>[]; eventType?: string } = { deals: [] }
60
+
61
+ while (reader.done === false) {
62
+ const { fieldNumber, wireType } = reader.readTag()
63
+
64
+ switch (fieldNumber) {
65
+ case 1:
66
+ message.deals.push(decodeTrade(reader.readBytes(wireType)))
67
+ break
68
+ case 2:
69
+ message.eventType = reader.readString(wireType)
70
+ break
71
+ default:
72
+ reader.skipField(wireType)
73
+ }
74
+ }
75
+
76
+ return message
77
+ }
78
+
79
+ function decodeTrade(data: Uint8Array) {
80
+ const reader = new ProtobufReader(data)
81
+ const message: { price?: string; quantity?: string; tradeType?: number; time?: string; tradeId?: string } = {}
82
+
83
+ while (reader.done === false) {
84
+ const { fieldNumber, wireType } = reader.readTag()
85
+
86
+ switch (fieldNumber) {
87
+ case 1:
88
+ message.price = reader.readString(wireType)
89
+ break
90
+ case 2:
91
+ message.quantity = reader.readString(wireType)
92
+ break
93
+ case 3:
94
+ message.tradeType = reader.readInt32(wireType)
95
+ break
96
+ case 4:
97
+ message.time = reader.readInt64String(wireType)
98
+ break
99
+ case 5:
100
+ message.tradeId = reader.readString(wireType)
101
+ break
102
+ default:
103
+ reader.skipField(wireType)
104
+ }
105
+ }
106
+
107
+ return message
108
+ }
109
+
110
+ function decodeDepths(data: Uint8Array) {
111
+ const reader = new ProtobufReader(data)
112
+ const message: {
113
+ asks: ReturnType<typeof decodePriceLevel>[]
114
+ bids: ReturnType<typeof decodePriceLevel>[]
115
+ eventType?: string
116
+ fromVersion?: string
117
+ toVersion?: string
118
+ lastOrderCreateTime?: string
119
+ } = { asks: [], bids: [] }
120
+
121
+ while (reader.done === false) {
122
+ const { fieldNumber, wireType } = reader.readTag()
123
+
124
+ switch (fieldNumber) {
125
+ case 1:
126
+ message.asks.push(decodePriceLevel(reader.readBytes(wireType)))
127
+ break
128
+ case 2:
129
+ message.bids.push(decodePriceLevel(reader.readBytes(wireType)))
130
+ break
131
+ case 3:
132
+ message.eventType = reader.readString(wireType)
133
+ break
134
+ case 4:
135
+ message.fromVersion = reader.readString(wireType)
136
+ break
137
+ case 5:
138
+ message.toVersion = reader.readString(wireType)
139
+ break
140
+ case 6:
141
+ message.lastOrderCreateTime = reader.readInt64String(wireType)
142
+ break
143
+ default:
144
+ reader.skipField(wireType)
145
+ }
146
+ }
147
+
148
+ return message
149
+ }
150
+
151
+ function decodePriceLevel(data: Uint8Array) {
152
+ const reader = new ProtobufReader(data)
153
+ const message: { price?: string; quantity?: string } = {}
154
+
155
+ while (reader.done === false) {
156
+ const { fieldNumber, wireType } = reader.readTag()
157
+
158
+ switch (fieldNumber) {
159
+ case 1:
160
+ message.price = reader.readString(wireType)
161
+ break
162
+ case 2:
163
+ message.quantity = reader.readString(wireType)
164
+ break
165
+ default:
166
+ reader.skipField(wireType)
167
+ }
168
+ }
169
+
170
+ return message
171
+ }
172
+
173
+ function decodeBookTicker(data: Uint8Array) {
174
+ const reader = new ProtobufReader(data)
175
+ const message: {
176
+ bidPrice?: string
177
+ bidQuantity?: string
178
+ askPrice?: string
179
+ askQuantity?: string
180
+ version?: string
181
+ lastOrderCreateTime?: string
182
+ } = {}
183
+
184
+ while (reader.done === false) {
185
+ const { fieldNumber, wireType } = reader.readTag()
186
+
187
+ switch (fieldNumber) {
188
+ case 1:
189
+ message.bidPrice = reader.readString(wireType)
190
+ break
191
+ case 2:
192
+ message.bidQuantity = reader.readString(wireType)
193
+ break
194
+ case 3:
195
+ message.askPrice = reader.readString(wireType)
196
+ break
197
+ case 4:
198
+ message.askQuantity = reader.readString(wireType)
199
+ break
200
+ case 5:
201
+ message.version = reader.readString(wireType)
202
+ break
203
+ case 6:
204
+ message.lastOrderCreateTime = reader.readInt64String(wireType)
205
+ break
206
+ default:
207
+ reader.skipField(wireType)
208
+ }
209
+ }
210
+
211
+ return message
212
+ }
213
+
214
+ class ProtobufReader {
215
+ private offset = 0
216
+
217
+ constructor(private readonly data: Uint8Array) {}
218
+
219
+ get done() {
220
+ return this.offset === this.data.length
221
+ }
222
+
223
+ readTag() {
224
+ const tag = this.readVarint()
225
+ if (typeof tag === 'bigint' || tag > 0xffffffff) {
226
+ throw new Error('Invalid protobuf tag')
227
+ }
228
+
229
+ const fieldNumber = tag >>> 3
230
+ const wireType = tag & 7
231
+
232
+ if (fieldNumber === 0) {
233
+ throw new Error('Invalid protobuf field number 0')
234
+ }
235
+
236
+ return { fieldNumber, wireType }
237
+ }
238
+
239
+ readString(wireType: number) {
240
+ this.expectWireType(wireType, 2)
241
+ const length = this.readLength()
242
+ const start = this.offset
243
+ const end = start + length
244
+
245
+ if (end > this.data.length) {
246
+ throw new Error('Invalid protobuf message length')
247
+ }
248
+
249
+ this.offset = end
250
+ return Buffer.isBuffer(this.data) ? this.data.toString('utf8', start, end) : textDecoder.decode(this.data.subarray(start, end))
251
+ }
252
+
253
+ readBytes(wireType: number) {
254
+ this.expectWireType(wireType, 2)
255
+ const length = this.readLength()
256
+ const end = this.offset + length
257
+
258
+ if (end > this.data.length) {
259
+ throw new Error('Invalid protobuf message length')
260
+ }
261
+
262
+ const value = this.data.subarray(this.offset, end)
263
+ this.offset = end
264
+ return value
265
+ }
266
+
267
+ readInt32(wireType: number) {
268
+ this.expectWireType(wireType, 0)
269
+ const value = this.readVarint()
270
+ return typeof value === 'number' ? value | 0 : Number(BigInt.asIntN(32, value))
271
+ }
272
+
273
+ readInt64String(wireType: number) {
274
+ this.expectWireType(wireType, 0)
275
+ const value = this.readVarint()
276
+ return typeof value === 'number' ? value.toString() : BigInt.asIntN(64, value).toString()
277
+ }
278
+
279
+ skipField(wireType: number): void {
280
+ switch (wireType) {
281
+ case 0:
282
+ this.skipVarint()
283
+ return
284
+ case 1:
285
+ this.skipBytes(8)
286
+ return
287
+ case 2:
288
+ this.skipBytes(this.readLength())
289
+ return
290
+ case 5:
291
+ this.skipBytes(4)
292
+ return
293
+ default:
294
+ throw new Error(`Unsupported protobuf wire type ${wireType}`)
295
+ }
296
+ }
297
+
298
+ private readVarint() {
299
+ let value = 0
300
+ let multiplier = 1
301
+
302
+ // Seven base-128 digits use at most 49 bits, so they remain exact and fast as a number.
303
+ for (let index = 0; index < 7; index++) {
304
+ const byte = this.readByte()
305
+ value += (byte & 0x7f) * multiplier
306
+
307
+ if ((byte & 0x80) === 0) {
308
+ return value
309
+ }
310
+ multiplier *= 0x80
311
+ }
312
+
313
+ // Larger int64 values need BigInt to preserve every bit before converting them to strings.
314
+ let bigValue = BigInt(value)
315
+ for (let index = 7; index < 10; index++) {
316
+ const byte = this.readByte()
317
+ bigValue |= BigInt(byte & 0x7f) << BigInt(index * 7)
318
+
319
+ if ((byte & 0x80) === 0) {
320
+ return bigValue
321
+ }
322
+ }
323
+
324
+ throw new Error('Invalid protobuf varint')
325
+ }
326
+
327
+ private readLength() {
328
+ const length = this.readVarint()
329
+ if (typeof length === 'bigint' || Number.isSafeInteger(length) === false) {
330
+ throw new Error('Invalid protobuf message length')
331
+ }
332
+ return length
333
+ }
334
+
335
+ private skipVarint() {
336
+ for (let index = 0; index < 10; index++) {
337
+ if ((this.readByte() & 0x80) === 0) {
338
+ return
339
+ }
340
+ }
341
+ throw new Error('Invalid protobuf varint')
342
+ }
343
+
344
+ private readByte() {
345
+ if (this.offset >= this.data.length) {
346
+ throw new Error('Unexpected end of protobuf message')
347
+ }
348
+ return this.data[this.offset++]!
349
+ }
350
+
351
+ private skipBytes(length: number) {
352
+ const end = this.offset + length
353
+ if (Number.isSafeInteger(length) === false || end > this.data.length) {
354
+ throw new Error('Invalid protobuf message length')
355
+ }
356
+ this.offset = end
357
+ }
358
+
359
+ private expectWireType(actual: number, expected: number) {
360
+ if (actual !== expected) {
361
+ throw new Error(`Invalid protobuf wire type ${actual}, expected ${expected}`)
362
+ }
363
+ }
364
+ }
@@ -1,8 +1,8 @@
1
- import dbg from 'debug'
2
1
  import WebSocket, { createWebSocketStream } from 'ws'
3
2
  import type { ClientRequestArgs } from 'http'
4
3
  import { PassThrough, Writable } from 'stream'
5
4
  import { setTimeout as sleep } from 'node:timers/promises'
5
+ import { createDebug, type DebugLogger } from '../debug.ts'
6
6
  import { getProxyAgent, ONE_SEC_IN_MS, optimizeFilters } from '../handy.ts'
7
7
  import { createManagedRealTimeIterator, mergeRealTime, type ManagedRealTimeIterator } from '../realtimeiterator.ts'
8
8
  import { Exchange, Filter } from '../types.ts'
@@ -34,7 +34,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
34
34
  return createManagedRealTimeIterator(this._stream(), () => this._close())
35
35
  }
36
36
 
37
- protected readonly debug: dbg.Debugger
37
+ protected readonly debug: DebugLogger
38
38
  protected abstract readonly wssURL: string
39
39
  protected readonly throttleSubscribeMS: number = 0
40
40
  protected readonly manualSnapshotsBuffer: any[] = []
@@ -53,7 +53,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
53
53
  private readonly _onError?: (error: Error) => void
54
54
  ) {
55
55
  this._filters = optimizeFilters(filters)
56
- this.debug = dbg(`tardis-dev:realtime:${_exchange}`)
56
+ this.debug = createDebug(`tardis-dev:realtime:${_exchange}`)
57
57
 
58
58
  this._wsClientOptions = {
59
59
  headers: {
@@ -479,7 +479,7 @@ export abstract class MultiConnectionRealTimeFeedBase implements RealTimeFeedIte
479
479
  }
480
480
 
481
481
  export abstract class PoolingClientBase implements RealTimeFeedIterable {
482
- protected readonly debug: dbg.Debugger
482
+ protected readonly debug: DebugLogger
483
483
  private _tid: NodeJS.Timeout | undefined = undefined
484
484
  private _outputStream: PassThrough | undefined
485
485
 
@@ -488,7 +488,7 @@ export abstract class PoolingClientBase implements RealTimeFeedIterable {
488
488
  private readonly _poolingIntervalSeconds: number,
489
489
  protected readonly onError?: (error: Error) => void
490
490
  ) {
491
- this.debug = dbg(`tardis-dev:pooling-client:${exchange}`)
491
+ this.debug = createDebug(`tardis-dev:pooling-client:${exchange}`)
492
492
  }
493
493
 
494
494
  [Symbol.asyncIterator](): ManagedRealTimeIterator<any> {
@@ -2,7 +2,7 @@ import { createReadStream } from 'node:fs'
2
2
  import { rm } from 'node:fs/promises'
3
3
  import { EventEmitter, once } from 'events'
4
4
  import { Worker } from 'worker_threads'
5
- import { constants, createGunzip, createZstdDecompress } from 'zlib'
5
+ import { constants, createGunzip } from 'zlib'
6
6
  import { BinarySplitBatchStream } from './binarysplit.ts'
7
7
  import { clearCacheSync } from './clearcache.ts'
8
8
  import { EXCHANGES, EXCHANGE_CHANNELS_INFO } from './consts.ts'
@@ -12,6 +12,7 @@ import { Mapper, MapperFactory, normalizeBookChanges } from './mappers/index.ts'
12
12
  import { getOptions } from './options.ts'
13
13
  import { Disconnect, Exchange, FilterForExchange } from './types.ts'
14
14
  import { WorkerJobPayload, WorkerMessage, WorkerSignal } from './worker.ts'
15
+ import { createZstdDecompressionStream } from './zstd.ts'
15
16
 
16
17
  type MapperOutput<T> = T extends MapperFactory<any, infer U> ? U : never
17
18
  type ReplayNormalizedMessage<U extends readonly MapperFactory<any, any>[], Z extends boolean> = Z extends true
@@ -170,9 +171,10 @@ async function* replayLineBatches<T extends Exchange>({
170
171
  // response is a path to file on disk let' read it as stream
171
172
  const { slicePath: cachedSlicePath, sliceSize } = cachedSlice
172
173
  const isZstdSlice = cachedSlicePath.endsWith('.zst')
173
- const linesStream = createReadStream(cachedSlicePath, { highWaterMark: CHUNK_SIZE })
174
- // decompress it while preserving the on-disk cache in the negotiated wire format
175
- .pipe(isZstdSlice ? createZstdDecompress({ chunkSize: CHUNK_SIZE }) : createGunzip(GZIP_OPTIONS))
174
+ const decompressedStream = isZstdSlice
175
+ ? createZstdDecompressionStream(cachedSlicePath, CHUNK_SIZE)
176
+ : createReadStream(cachedSlicePath, { highWaterMark: CHUNK_SIZE }).pipe(createGunzip(GZIP_OPTIONS))
177
+ const linesStream = decompressedStream
176
178
  .on('error', function onDecompressionError(err) {
177
179
  debug('%s decompression error %o', isZstdSlice ? 'zstd' : 'gzip', err)
178
180
  linesStream.destroy(err)
@@ -1,13 +1,14 @@
1
- import dbg from 'debug'
2
1
  import { existsSync } from 'node:fs'
3
2
  import { isMainThread, parentPort, workerData } from 'worker_threads'
3
+ import { createDebug } from './debug.ts'
4
4
  import { forEachConcurrent } from './foreachconcurrent.ts'
5
5
  import { addMinutes, download, formatDateToPath, optimizeFilters, sha256, wait, cleanTempFiles } from './handy.ts'
6
6
  import type { DataFeedCompression } from './options.ts'
7
7
  import { Exchange, Filter } from './types.ts'
8
- const debug = dbg('tardis-dev')
8
+ const debug = createDebug('tardis-dev')
9
9
 
10
10
  const DEFAULT_DATA_FEED_SLICE_SIZE = 1
11
+ const DATA_FEED_DOWNLOAD_ATTEMPT_TIMEOUT_MS = 135_000
11
12
 
12
13
  if (isMainThread) {
13
14
  debug('current worker is not meant to run in main thread')
@@ -162,7 +163,8 @@ async function getDataFeedSlice(
162
163
  }
163
164
  }
164
165
 
165
- let url = `${endpoint}/data-feeds/${exchange}?from=${fromDate.toISOString()}&offset=${offset}&compression=${dataFeedCompression}`
166
+ const requestCompression = dataFeedCompression === 'zstd' ? 'zstd-multiframe' : dataFeedCompression
167
+ let url = `${endpoint}/data-feeds/${exchange}?from=${fromDate.toISOString()}&offset=${offset}&compression=${requestCompression}`
166
168
  if (requestedSliceSize > DEFAULT_DATA_FEED_SLICE_SIZE) {
167
169
  url += `&sliceSize=${requestedSliceSize}`
168
170
  }
@@ -177,7 +179,8 @@ async function getDataFeedSlice(
177
179
  url,
178
180
  userAgent,
179
181
  appendContentEncodingExtension: true,
180
- acceptEncoding: dataFeedCompression === 'gzip' ? 'gzip' : 'zstd, gzip'
182
+ acceptEncoding: dataFeedCompression === 'gzip' ? 'gzip' : 'zstd, gzip',
183
+ attemptTimeoutMS: DATA_FEED_DOWNLOAD_ATTEMPT_TIMEOUT_MS
181
184
  })
182
185
  const responseSliceSize = Number(downloadResult.headers['x-slice-size'])
183
186
  const suggestedSliceSize = Number(downloadResult.headers['x-suggested-slice-size'] ?? DEFAULT_DATA_FEED_SLICE_SIZE)
@@ -0,0 +1,139 @@
1
+ import { once } from 'node:events'
2
+ import { open, type FileHandle } from 'node:fs/promises'
3
+ import { Readable } from 'node:stream'
4
+ import { createZstdDecompress } from 'node:zlib'
5
+
6
+ const ZSTD_FRAME_MAGIC = 0xfd2fb528
7
+ const DICTIONARY_ID_SIZES = [0, 1, 2, 4]
8
+
9
+ export function createZstdDecompressionStream(path: string, chunkSize: number) {
10
+ return Readable.from(decompressZstdFrames(path, chunkSize), { objectMode: false })
11
+ }
12
+
13
+ async function* decompressZstdFrames(path: string, chunkSize: number) {
14
+ const file = await open(path, 'r')
15
+ try {
16
+ const fileSize = (await file.stat()).size
17
+ const frames = await findZstdFrames(file, path, fileSize)
18
+
19
+ for (const frame of frames) {
20
+ const source = Readable.from(readFileRange(file, frame.offset, frame.length, chunkSize), { objectMode: false })
21
+ const decompressor = createZstdDecompress({ chunkSize })
22
+ source.on('error', (error) => decompressor.destroy(error))
23
+ source.pipe(decompressor)
24
+
25
+ try {
26
+ for await (const chunk of decompressor) {
27
+ yield chunk
28
+ }
29
+
30
+ if (decompressor.bytesWritten !== frame.length) {
31
+ throw new Error(
32
+ `Zstd decoder consumed ${decompressor.bytesWritten} of ${frame.length} bytes at offset ${frame.offset} in ${path}`
33
+ )
34
+ }
35
+ } finally {
36
+ source.unpipe(decompressor)
37
+ source.destroy()
38
+ decompressor.destroy()
39
+ if (!source.closed) {
40
+ await once(source, 'close')
41
+ }
42
+ }
43
+ }
44
+ } finally {
45
+ await file.close()
46
+ }
47
+ }
48
+
49
+ async function findZstdFrames(file: FileHandle, path: string, fileSize: number) {
50
+ if (fileSize === 0) {
51
+ throw new Error(`Empty zstd file: ${path}`)
52
+ }
53
+
54
+ const bytes = Buffer.allocUnsafe(4)
55
+ const frames: { offset: number; length: number }[] = []
56
+ let offset = 0
57
+
58
+ while (offset < fileSize) {
59
+ const frameStart = offset
60
+ await readExactly(file, bytes, 4, offset, fileSize, path)
61
+ if (bytes.readUInt32LE(0) !== ZSTD_FRAME_MAGIC) {
62
+ throw new Error(`Invalid zstd frame magic at offset ${offset} in ${path}`)
63
+ }
64
+ offset += 4
65
+
66
+ await readExactly(file, bytes, 1, offset, fileSize, path)
67
+ const descriptor = bytes[0]
68
+ if ((descriptor & 0x08) !== 0) {
69
+ throw new Error(`Unsupported reserved zstd frame descriptor bit at offset ${offset} in ${path}`)
70
+ }
71
+ offset++
72
+
73
+ const singleSegment = (descriptor & 0x20) !== 0
74
+ const contentChecksum = (descriptor & 0x04) !== 0
75
+ const dictionaryIdSize = DICTIONARY_ID_SIZES[descriptor & 0x03]
76
+ const contentSizeFlag = descriptor >>> 6
77
+ const contentSizeSize = contentSizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << contentSizeFlag
78
+ const windowDescriptorSize = singleSegment ? 0 : 1
79
+ offset += windowDescriptorSize + dictionaryIdSize + contentSizeSize
80
+ ensureAvailable(offset, 0, fileSize, path)
81
+
82
+ let lastBlock = false
83
+ while (!lastBlock) {
84
+ await readExactly(file, bytes, 3, offset, fileSize, path)
85
+ const blockHeader = bytes.readUIntLE(0, 3)
86
+ offset += 3
87
+ lastBlock = (blockHeader & 1) !== 0
88
+ const blockType = (blockHeader >>> 1) & 3
89
+ if (blockType === 3) {
90
+ throw new Error(`Invalid reserved zstd block type at offset ${offset - 3} in ${path}`)
91
+ }
92
+
93
+ const blockSize = blockHeader >>> 3
94
+ // RLE blocks store one repeated byte; other block types store blockSize bytes.
95
+ offset += blockType === 1 ? 1 : blockSize
96
+ ensureAvailable(offset, 0, fileSize, path)
97
+ }
98
+
99
+ if (contentChecksum) {
100
+ offset += 4
101
+ ensureAvailable(offset, 0, fileSize, path)
102
+ }
103
+ frames.push({ offset: frameStart, length: offset - frameStart })
104
+ }
105
+
106
+ return frames
107
+ }
108
+
109
+ async function* readFileRange(file: FileHandle, position: number, length: number, chunkSize: number) {
110
+ let bytesRemaining = length
111
+ while (bytesRemaining > 0) {
112
+ const buffer = Buffer.allocUnsafe(Math.min(bytesRemaining, chunkSize))
113
+ const { bytesRead } = await file.read(buffer, 0, buffer.length, position)
114
+ if (bytesRead === 0) {
115
+ throw new Error(`Expected ${length} compressed bytes, read ${length - bytesRemaining}`)
116
+ }
117
+ yield bytesRead === buffer.length ? buffer : buffer.subarray(0, bytesRead)
118
+ position += bytesRead
119
+ bytesRemaining -= bytesRead
120
+ }
121
+ }
122
+
123
+ async function readExactly(file: FileHandle, buffer: Buffer, length: number, position: number, fileSize: number, path: string) {
124
+ ensureAvailable(position, length, fileSize, path)
125
+ let bytesRead = 0
126
+ while (bytesRead < length) {
127
+ const result = await file.read(buffer, bytesRead, length - bytesRead, position + bytesRead)
128
+ if (result.bytesRead === 0) {
129
+ throw new Error(`Truncated zstd frame at offset ${position + bytesRead} in ${path}`)
130
+ }
131
+ bytesRead += result.bytesRead
132
+ }
133
+ }
134
+
135
+ function ensureAvailable(position: number, length: number, fileSize: number, path: string) {
136
+ if (position < 0 || length < 0 || position > fileSize - length) {
137
+ throw new Error(`Truncated zstd frame at offset ${position} in ${path}`)
138
+ }
139
+ }
package/package.json CHANGED
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "name": "tardis-machine",
3
- "version": "18.0.0",
3
+ "version": "18.1.0",
4
4
  "engines": {
5
- "node": ">=24.5"
5
+ "node": ">=24.19"
6
6
  },
7
7
  "devEngines": {
8
8
  "runtime": {
9
9
  "name": "node",
10
- "version": "^24.15.0 || >=26.0.0"
10
+ "version": "^24.19.0 || >=26.0.0"
11
11
  },
12
12
  "packageManager": {
13
13
  "name": "npm",
@@ -31,7 +31,8 @@
31
31
  "homepage": "https://github.com/tardis-dev/tardis-machine",
32
32
  "scripts": {
33
33
  "build": "tsc",
34
- "test": "npm run build && cross-env UWS_HTTP_MAX_HEADERS_SIZE=20000 node --experimental-vm-modules ./node_modules/jest/bin/jest.js --forceExit --runInBand",
34
+ "test": "npm run build && tsc -p test/tsconfig.json && node --test test/*.test.ts",
35
+ "test:update-snapshots": "npm run build && tsc -p test/tsconfig.json && node --test --test-update-snapshots test/*.test.ts",
35
36
  "prepare": "npm run build",
36
37
  "format": "prettier --write .",
37
38
  "check-format": "prettier --check .",
@@ -74,43 +75,16 @@
74
75
  ],
75
76
  "license": "MPL-2.0",
76
77
  "dependencies": {
77
- "debug": "^4.4.1",
78
- "tardis-dev": "^18.0.1",
79
- "uWebSockets.js": "git+https://github.com/uNetworking/uWebSockets.js.git#dddd8ffd1b2c28a66022160923ca92f064cdacb4"
78
+ "debug": "^4.4.3",
79
+ "tardis-dev": "^18.1.0",
80
+ "ws": "^8.21.1"
80
81
  },
81
82
  "bundleDependencies": true,
82
83
  "devDependencies": {
83
84
  "@types/debug": "^4.1.12",
84
- "@types/jest": "^30.0.0",
85
85
  "@types/node": "^26.1.2",
86
- "@types/node-fetch": "^2.6.13",
87
- "@types/split2": "^4.2.3",
88
86
  "@types/ws": "^8.18.1",
89
- "cross-env": "^10.1.0",
90
- "jest": "^30.0.5",
91
- "node-fetch": "^2.6.1",
92
87
  "prettier": "^3.8.3",
93
- "split2": "^4.2.0",
94
- "ts-jest": "^29.4.0",
95
- "typescript": "^5.9.2",
96
- "ws": "^8.18.3"
97
- },
98
- "jest": {
99
- "extensionsToTreatAsEsm": [
100
- ".ts"
101
- ],
102
- "moduleNameMapper": {
103
- "^(\\.{1,2}/.*)\\.js$": "$1"
104
- },
105
- "transform": {
106
- "\\.(ts|tsx)?$": [
107
- "ts-jest",
108
- {
109
- "useESM": true,
110
- "tsconfig": "./test/tsconfig.json"
111
- }
112
- ]
113
- },
114
- "testEnvironment": "node"
88
+ "typescript": "^6.0.3"
115
89
  }
116
90
  }