tardis-dev 18.0.0 → 18.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +1 -1
  2. package/dist/combine.js +2 -5
  3. package/dist/combine.js.map +1 -1
  4. package/dist/debug.d.ts +12 -2
  5. package/dist/debug.d.ts.map +1 -1
  6. package/dist/debug.js +2 -1
  7. package/dist/debug.js.map +1 -1
  8. package/dist/downloaddatasets.d.ts.map +1 -1
  9. package/dist/downloaddatasets.js +4 -4
  10. package/dist/downloaddatasets.js.map +1 -1
  11. package/dist/foreachconcurrent.d.ts +2 -0
  12. package/dist/foreachconcurrent.d.ts.map +1 -0
  13. package/dist/foreachconcurrent.js +28 -0
  14. package/dist/foreachconcurrent.js.map +1 -0
  15. package/dist/handy.d.ts +3 -1
  16. package/dist/handy.d.ts.map +1 -1
  17. package/dist/handy.js +72 -86
  18. package/dist/handy.js.map +1 -1
  19. package/dist/realtimefeeds/mexc.d.ts +0 -7
  20. package/dist/realtimefeeds/mexc.d.ts.map +1 -1
  21. package/dist/realtimefeeds/mexc.js +2 -70
  22. package/dist/realtimefeeds/mexc.js.map +1 -1
  23. package/dist/realtimefeeds/mexcprotobuf.d.ts +43 -0
  24. package/dist/realtimefeeds/mexcprotobuf.d.ts.map +1 -0
  25. package/dist/realtimefeeds/mexcprotobuf.js +294 -0
  26. package/dist/realtimefeeds/mexcprotobuf.js.map +1 -0
  27. package/dist/realtimefeeds/realtimefeed.d.ts +3 -3
  28. package/dist/realtimefeeds/realtimefeed.d.ts.map +1 -1
  29. package/dist/realtimefeeds/realtimefeed.js +3 -3
  30. package/dist/realtimefeeds/realtimefeed.js.map +1 -1
  31. package/dist/worker.d.ts.map +1 -1
  32. package/dist/worker.js +5 -5
  33. package/dist/worker.js.map +1 -1
  34. package/package.json +3 -8
  35. package/src/combine.ts +2 -4
  36. package/src/debug.ts +16 -1
  37. package/src/downloaddatasets.ts +4 -4
  38. package/src/external-modules.d.ts +5 -0
  39. package/src/foreachconcurrent.ts +29 -0
  40. package/src/handy.ts +93 -89
  41. package/src/realtimefeeds/mexc.ts +2 -72
  42. package/src/realtimefeeds/mexcprotobuf.ts +364 -0
  43. package/src/realtimefeeds/realtimefeed.ts +5 -5
  44. package/src/worker.ts +8 -12
@@ -0,0 +1,364 @@
1
+ const textDecoder = new TextDecoder()
2
+
3
+ // Decodes only the current MEXC real-time channels supported by MexcRealTimeFeed.
4
+ // Field numbers mirror the official schemas at commit 7b8ac7a6681f28551612a5a7cefbb7e09b56bb85:
5
+ // https://github.com/mexcdevelop/websocket-proto/tree/7b8ac7a6681f28551612a5a7cefbb7e09b56bb85
6
+ // int64 values remain decimal strings and repeated fields remain empty arrays when absent,
7
+ // preserving the output shape previously produced by protobufjs.
8
+ export function decodeMexcProtobufMessage(data: Uint8Array) {
9
+ const reader = new ProtobufReader(data)
10
+ const message: {
11
+ channel?: string
12
+ symbol?: string
13
+ symbolId?: string
14
+ createTime?: string
15
+ sendTime?: string
16
+ publicAggreDepths?: ReturnType<typeof decodeDepths>
17
+ publicAggreDeals?: ReturnType<typeof decodeDeals>
18
+ publicAggreBookTicker?: ReturnType<typeof decodeBookTicker>
19
+ } = {}
20
+
21
+ while (reader.done === false) {
22
+ const { fieldNumber, wireType } = reader.readTag()
23
+
24
+ switch (fieldNumber) {
25
+ case 1:
26
+ message.channel = reader.readString(wireType)
27
+ break
28
+ case 3:
29
+ message.symbol = reader.readString(wireType)
30
+ break
31
+ case 4:
32
+ message.symbolId = reader.readString(wireType)
33
+ break
34
+ case 5:
35
+ message.createTime = reader.readInt64String(wireType)
36
+ break
37
+ case 6:
38
+ message.sendTime = reader.readInt64String(wireType)
39
+ break
40
+ case 313:
41
+ message.publicAggreDepths = decodeDepths(reader.readBytes(wireType))
42
+ break
43
+ case 314:
44
+ message.publicAggreDeals = decodeDeals(reader.readBytes(wireType))
45
+ break
46
+ case 315:
47
+ message.publicAggreBookTicker = decodeBookTicker(reader.readBytes(wireType))
48
+ break
49
+ default:
50
+ reader.skipField(wireType)
51
+ }
52
+ }
53
+
54
+ return message
55
+ }
56
+
57
+ function decodeDeals(data: Uint8Array) {
58
+ const reader = new ProtobufReader(data)
59
+ const message: { deals: ReturnType<typeof decodeTrade>[]; eventType?: string } = { deals: [] }
60
+
61
+ while (reader.done === false) {
62
+ const { fieldNumber, wireType } = reader.readTag()
63
+
64
+ switch (fieldNumber) {
65
+ case 1:
66
+ message.deals.push(decodeTrade(reader.readBytes(wireType)))
67
+ break
68
+ case 2:
69
+ message.eventType = reader.readString(wireType)
70
+ break
71
+ default:
72
+ reader.skipField(wireType)
73
+ }
74
+ }
75
+
76
+ return message
77
+ }
78
+
79
+ function decodeTrade(data: Uint8Array) {
80
+ const reader = new ProtobufReader(data)
81
+ const message: { price?: string; quantity?: string; tradeType?: number; time?: string; tradeId?: string } = {}
82
+
83
+ while (reader.done === false) {
84
+ const { fieldNumber, wireType } = reader.readTag()
85
+
86
+ switch (fieldNumber) {
87
+ case 1:
88
+ message.price = reader.readString(wireType)
89
+ break
90
+ case 2:
91
+ message.quantity = reader.readString(wireType)
92
+ break
93
+ case 3:
94
+ message.tradeType = reader.readInt32(wireType)
95
+ break
96
+ case 4:
97
+ message.time = reader.readInt64String(wireType)
98
+ break
99
+ case 5:
100
+ message.tradeId = reader.readString(wireType)
101
+ break
102
+ default:
103
+ reader.skipField(wireType)
104
+ }
105
+ }
106
+
107
+ return message
108
+ }
109
+
110
+ function decodeDepths(data: Uint8Array) {
111
+ const reader = new ProtobufReader(data)
112
+ const message: {
113
+ asks: ReturnType<typeof decodePriceLevel>[]
114
+ bids: ReturnType<typeof decodePriceLevel>[]
115
+ eventType?: string
116
+ fromVersion?: string
117
+ toVersion?: string
118
+ lastOrderCreateTime?: string
119
+ } = { asks: [], bids: [] }
120
+
121
+ while (reader.done === false) {
122
+ const { fieldNumber, wireType } = reader.readTag()
123
+
124
+ switch (fieldNumber) {
125
+ case 1:
126
+ message.asks.push(decodePriceLevel(reader.readBytes(wireType)))
127
+ break
128
+ case 2:
129
+ message.bids.push(decodePriceLevel(reader.readBytes(wireType)))
130
+ break
131
+ case 3:
132
+ message.eventType = reader.readString(wireType)
133
+ break
134
+ case 4:
135
+ message.fromVersion = reader.readString(wireType)
136
+ break
137
+ case 5:
138
+ message.toVersion = reader.readString(wireType)
139
+ break
140
+ case 6:
141
+ message.lastOrderCreateTime = reader.readInt64String(wireType)
142
+ break
143
+ default:
144
+ reader.skipField(wireType)
145
+ }
146
+ }
147
+
148
+ return message
149
+ }
150
+
151
+ function decodePriceLevel(data: Uint8Array) {
152
+ const reader = new ProtobufReader(data)
153
+ const message: { price?: string; quantity?: string } = {}
154
+
155
+ while (reader.done === false) {
156
+ const { fieldNumber, wireType } = reader.readTag()
157
+
158
+ switch (fieldNumber) {
159
+ case 1:
160
+ message.price = reader.readString(wireType)
161
+ break
162
+ case 2:
163
+ message.quantity = reader.readString(wireType)
164
+ break
165
+ default:
166
+ reader.skipField(wireType)
167
+ }
168
+ }
169
+
170
+ return message
171
+ }
172
+
173
+ function decodeBookTicker(data: Uint8Array) {
174
+ const reader = new ProtobufReader(data)
175
+ const message: {
176
+ bidPrice?: string
177
+ bidQuantity?: string
178
+ askPrice?: string
179
+ askQuantity?: string
180
+ version?: string
181
+ lastOrderCreateTime?: string
182
+ } = {}
183
+
184
+ while (reader.done === false) {
185
+ const { fieldNumber, wireType } = reader.readTag()
186
+
187
+ switch (fieldNumber) {
188
+ case 1:
189
+ message.bidPrice = reader.readString(wireType)
190
+ break
191
+ case 2:
192
+ message.bidQuantity = reader.readString(wireType)
193
+ break
194
+ case 3:
195
+ message.askPrice = reader.readString(wireType)
196
+ break
197
+ case 4:
198
+ message.askQuantity = reader.readString(wireType)
199
+ break
200
+ case 5:
201
+ message.version = reader.readString(wireType)
202
+ break
203
+ case 6:
204
+ message.lastOrderCreateTime = reader.readInt64String(wireType)
205
+ break
206
+ default:
207
+ reader.skipField(wireType)
208
+ }
209
+ }
210
+
211
+ return message
212
+ }
213
+
214
+ class ProtobufReader {
215
+ private offset = 0
216
+
217
+ constructor(private readonly data: Uint8Array) {}
218
+
219
+ get done() {
220
+ return this.offset === this.data.length
221
+ }
222
+
223
+ readTag() {
224
+ const tag = this.readVarint()
225
+ if (typeof tag === 'bigint' || tag > 0xffffffff) {
226
+ throw new Error('Invalid protobuf tag')
227
+ }
228
+
229
+ const fieldNumber = tag >>> 3
230
+ const wireType = tag & 7
231
+
232
+ if (fieldNumber === 0) {
233
+ throw new Error('Invalid protobuf field number 0')
234
+ }
235
+
236
+ return { fieldNumber, wireType }
237
+ }
238
+
239
+ readString(wireType: number) {
240
+ this.expectWireType(wireType, 2)
241
+ const length = this.readLength()
242
+ const start = this.offset
243
+ const end = start + length
244
+
245
+ if (end > this.data.length) {
246
+ throw new Error('Invalid protobuf message length')
247
+ }
248
+
249
+ this.offset = end
250
+ return Buffer.isBuffer(this.data) ? this.data.toString('utf8', start, end) : textDecoder.decode(this.data.subarray(start, end))
251
+ }
252
+
253
+ readBytes(wireType: number) {
254
+ this.expectWireType(wireType, 2)
255
+ const length = this.readLength()
256
+ const end = this.offset + length
257
+
258
+ if (end > this.data.length) {
259
+ throw new Error('Invalid protobuf message length')
260
+ }
261
+
262
+ const value = this.data.subarray(this.offset, end)
263
+ this.offset = end
264
+ return value
265
+ }
266
+
267
+ readInt32(wireType: number) {
268
+ this.expectWireType(wireType, 0)
269
+ const value = this.readVarint()
270
+ return typeof value === 'number' ? value | 0 : Number(BigInt.asIntN(32, value))
271
+ }
272
+
273
+ readInt64String(wireType: number) {
274
+ this.expectWireType(wireType, 0)
275
+ const value = this.readVarint()
276
+ return typeof value === 'number' ? value.toString() : BigInt.asIntN(64, value).toString()
277
+ }
278
+
279
+ skipField(wireType: number): void {
280
+ switch (wireType) {
281
+ case 0:
282
+ this.skipVarint()
283
+ return
284
+ case 1:
285
+ this.skipBytes(8)
286
+ return
287
+ case 2:
288
+ this.skipBytes(this.readLength())
289
+ return
290
+ case 5:
291
+ this.skipBytes(4)
292
+ return
293
+ default:
294
+ throw new Error(`Unsupported protobuf wire type ${wireType}`)
295
+ }
296
+ }
297
+
298
+ private readVarint() {
299
+ let value = 0
300
+ let multiplier = 1
301
+
302
+ // Seven base-128 digits use at most 49 bits, so they remain exact and fast as a number.
303
+ for (let index = 0; index < 7; index++) {
304
+ const byte = this.readByte()
305
+ value += (byte & 0x7f) * multiplier
306
+
307
+ if ((byte & 0x80) === 0) {
308
+ return value
309
+ }
310
+ multiplier *= 0x80
311
+ }
312
+
313
+ // Larger int64 values need BigInt to preserve every bit before converting them to strings.
314
+ let bigValue = BigInt(value)
315
+ for (let index = 7; index < 10; index++) {
316
+ const byte = this.readByte()
317
+ bigValue |= BigInt(byte & 0x7f) << BigInt(index * 7)
318
+
319
+ if ((byte & 0x80) === 0) {
320
+ return bigValue
321
+ }
322
+ }
323
+
324
+ throw new Error('Invalid protobuf varint')
325
+ }
326
+
327
+ private readLength() {
328
+ const length = this.readVarint()
329
+ if (typeof length === 'bigint' || Number.isSafeInteger(length) === false) {
330
+ throw new Error('Invalid protobuf message length')
331
+ }
332
+ return length
333
+ }
334
+
335
+ private skipVarint() {
336
+ for (let index = 0; index < 10; index++) {
337
+ if ((this.readByte() & 0x80) === 0) {
338
+ return
339
+ }
340
+ }
341
+ throw new Error('Invalid protobuf varint')
342
+ }
343
+
344
+ private readByte() {
345
+ if (this.offset >= this.data.length) {
346
+ throw new Error('Unexpected end of protobuf message')
347
+ }
348
+ return this.data[this.offset++]!
349
+ }
350
+
351
+ private skipBytes(length: number) {
352
+ const end = this.offset + length
353
+ if (Number.isSafeInteger(length) === false || end > this.data.length) {
354
+ throw new Error('Invalid protobuf message length')
355
+ }
356
+ this.offset = end
357
+ }
358
+
359
+ private expectWireType(actual: number, expected: number) {
360
+ if (actual !== expected) {
361
+ throw new Error(`Invalid protobuf wire type ${actual}, expected ${expected}`)
362
+ }
363
+ }
364
+ }
@@ -1,8 +1,8 @@
1
- import dbg from 'debug'
2
1
  import WebSocket, { createWebSocketStream } from 'ws'
3
2
  import type { ClientRequestArgs } from 'http'
4
3
  import { PassThrough, Writable } from 'stream'
5
4
  import { setTimeout as sleep } from 'node:timers/promises'
5
+ import { createDebug, type DebugLogger } from '../debug.ts'
6
6
  import { getProxyAgent, ONE_SEC_IN_MS, optimizeFilters } from '../handy.ts'
7
7
  import { createManagedRealTimeIterator, mergeRealTime, type ManagedRealTimeIterator } from '../realtimeiterator.ts'
8
8
  import { Exchange, Filter } from '../types.ts'
@@ -34,7 +34,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
34
34
  return createManagedRealTimeIterator(this._stream(), () => this._close())
35
35
  }
36
36
 
37
- protected readonly debug: dbg.Debugger
37
+ protected readonly debug: DebugLogger
38
38
  protected abstract readonly wssURL: string
39
39
  protected readonly throttleSubscribeMS: number = 0
40
40
  protected readonly manualSnapshotsBuffer: any[] = []
@@ -53,7 +53,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
53
53
  private readonly _onError?: (error: Error) => void
54
54
  ) {
55
55
  this._filters = optimizeFilters(filters)
56
- this.debug = dbg(`tardis-dev:realtime:${_exchange}`)
56
+ this.debug = createDebug(`tardis-dev:realtime:${_exchange}`)
57
57
 
58
58
  this._wsClientOptions = {
59
59
  headers: {
@@ -479,7 +479,7 @@ export abstract class MultiConnectionRealTimeFeedBase implements RealTimeFeedIte
479
479
  }
480
480
 
481
481
  export abstract class PoolingClientBase implements RealTimeFeedIterable {
482
- protected readonly debug: dbg.Debugger
482
+ protected readonly debug: DebugLogger
483
483
  private _tid: NodeJS.Timeout | undefined = undefined
484
484
  private _outputStream: PassThrough | undefined
485
485
 
@@ -488,7 +488,7 @@ export abstract class PoolingClientBase implements RealTimeFeedIterable {
488
488
  private readonly _poolingIntervalSeconds: number,
489
489
  protected readonly onError?: (error: Error) => void
490
490
  ) {
491
- this.debug = dbg(`tardis-dev:pooling-client:${exchange}`)
491
+ this.debug = createDebug(`tardis-dev:pooling-client:${exchange}`)
492
492
  }
493
493
 
494
494
  [Symbol.asyncIterator](): ManagedRealTimeIterator<any> {
package/src/worker.ts CHANGED
@@ -1,11 +1,11 @@
1
- import dbg from 'debug'
2
1
  import { existsSync } from 'node:fs'
3
- import pMap from 'p-map'
4
2
  import { isMainThread, parentPort, workerData } from 'worker_threads'
5
- import { addMinutes, download, formatDateToPath, optimizeFilters, sequence, sha256, wait, cleanTempFiles } from './handy.ts'
3
+ import { createDebug } from './debug.ts'
4
+ import { forEachConcurrent } from './foreachconcurrent.ts'
5
+ import { addMinutes, download, formatDateToPath, optimizeFilters, sha256, wait, cleanTempFiles } from './handy.ts'
6
6
  import type { DataFeedCompression } from './options.ts'
7
7
  import { Exchange, Filter } from './types.ts'
8
- const debug = dbg('tardis-dev')
8
+ const debug = createDebug('tardis-dev')
9
9
 
10
10
  const DEFAULT_DATA_FEED_SLICE_SIZE = 1
11
11
 
@@ -123,14 +123,10 @@ async function getAvailableDataFeedSlices(
123
123
  }
124
124
 
125
125
  // it both begining and end date of the range is accessible fetch all remaning slices concurently with CONCURRENCY_LIMIT
126
- await pMap(
127
- sliceOffsets,
128
- async (offset) => {
129
- const requestedSliceSize = Math.min(replaySliceSize, minutesCountToFetch - 1 - offset)
130
- await getDataFeedSlice(payload, offset, filters, cacheDir, requestedSliceSize)
131
- },
132
- { concurrency: concurrencyLimit }
133
- )
126
+ await forEachConcurrent(sliceOffsets, concurrencyLimit, async (offset) => {
127
+ const requestedSliceSize = Math.min(replaySliceSize, minutesCountToFetch - 1 - offset)
128
+ await getDataFeedSlice(payload, offset, filters, cacheDir, requestedSliceSize)
129
+ })
134
130
  }
135
131
 
136
132
  async function getDataFeedSlice(