tardis-machine 18.2.1 → 18.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/dist/ws/subscriptionsmappers.d.ts.map +1 -1
  2. package/dist/ws/subscriptionsmappers.js +17 -0
  3. package/dist/ws/subscriptionsmappers.js.map +1 -1
  4. package/node_modules/tardis-dev/README.md +3 -1
  5. package/node_modules/tardis-dev/dist/consts.d.ts +4 -1
  6. package/node_modules/tardis-dev/dist/consts.d.ts.map +1 -1
  7. package/node_modules/tardis-dev/dist/consts.js +8 -0
  8. package/node_modules/tardis-dev/dist/consts.js.map +1 -1
  9. package/node_modules/tardis-dev/dist/mappers/ascendex.d.ts +4 -4
  10. package/node_modules/tardis-dev/dist/mappers/aster.d.ts +8 -8
  11. package/node_modules/tardis-dev/dist/mappers/binance.d.ts +18 -18
  12. package/node_modules/tardis-dev/dist/mappers/binanceeuropeanoptions.d.ts +1 -1
  13. package/node_modules/tardis-dev/dist/mappers/bitfinex.d.ts +8 -8
  14. package/node_modules/tardis-dev/dist/mappers/bitflyer.d.ts +1 -1
  15. package/node_modules/tardis-dev/dist/mappers/bitget.d.ts +1 -1
  16. package/node_modules/tardis-dev/dist/mappers/bithumb.d.ts +8 -0
  17. package/node_modules/tardis-dev/dist/mappers/bithumb.d.ts.map +1 -0
  18. package/node_modules/tardis-dev/dist/mappers/bithumb.js +60 -0
  19. package/node_modules/tardis-dev/dist/mappers/bithumb.js.map +1 -0
  20. package/node_modules/tardis-dev/dist/mappers/bitmex.d.ts +2 -2
  21. package/node_modules/tardis-dev/dist/mappers/bitnomial.d.ts +1 -1
  22. package/node_modules/tardis-dev/dist/mappers/bitstamp.d.ts +1 -1
  23. package/node_modules/tardis-dev/dist/mappers/bitvavo.d.ts +9 -0
  24. package/node_modules/tardis-dev/dist/mappers/bitvavo.d.ts.map +1 -0
  25. package/node_modules/tardis-dev/dist/mappers/bitvavo.js +168 -0
  26. package/node_modules/tardis-dev/dist/mappers/bitvavo.js.map +1 -0
  27. package/node_modules/tardis-dev/dist/mappers/blockchaincom.d.ts +2 -2
  28. package/node_modules/tardis-dev/dist/mappers/bullish.d.ts +5 -5
  29. package/node_modules/tardis-dev/dist/mappers/bybit.d.ts +7 -7
  30. package/node_modules/tardis-dev/dist/mappers/bybitspot.d.ts +1 -1
  31. package/node_modules/tardis-dev/dist/mappers/coinbase.d.ts +1 -1
  32. package/node_modules/tardis-dev/dist/mappers/coinbaseinternational.d.ts +2 -2
  33. package/node_modules/tardis-dev/dist/mappers/coinflex.d.ts +1 -1
  34. package/node_modules/tardis-dev/dist/mappers/cryptocom.d.ts +5 -5
  35. package/node_modules/tardis-dev/dist/mappers/delta.d.ts +1 -1
  36. package/node_modules/tardis-dev/dist/mappers/deribit.d.ts +3 -3
  37. package/node_modules/tardis-dev/dist/mappers/dydx.d.ts +3 -3
  38. package/node_modules/tardis-dev/dist/mappers/dydxv4.d.ts +4 -4
  39. package/node_modules/tardis-dev/dist/mappers/ftx.d.ts +8 -8
  40. package/node_modules/tardis-dev/dist/mappers/gateio.d.ts +1 -1
  41. package/node_modules/tardis-dev/dist/mappers/gateiofutures.d.ts +4 -4
  42. package/node_modules/tardis-dev/dist/mappers/gemini.d.ts +1 -1
  43. package/node_modules/tardis-dev/dist/mappers/huobi.d.ts +26 -26
  44. package/node_modules/tardis-dev/dist/mappers/hyperliquid.d.ts +3 -3
  45. package/node_modules/tardis-dev/dist/mappers/index.d.ts +200 -170
  46. package/node_modules/tardis-dev/dist/mappers/index.d.ts.map +1 -1
  47. package/node_modules/tardis-dev/dist/mappers/index.js +3 -1
  48. package/node_modules/tardis-dev/dist/mappers/index.js.map +1 -1
  49. package/node_modules/tardis-dev/dist/mappers/kucoin.d.ts +3 -3
  50. package/node_modules/tardis-dev/dist/mappers/kucoinfutures.d.ts +4 -4
  51. package/node_modules/tardis-dev/dist/mappers/lighter.d.ts +164 -5
  52. package/node_modules/tardis-dev/dist/mappers/lighter.d.ts.map +1 -1
  53. package/node_modules/tardis-dev/dist/mappers/lighter.js +41 -15
  54. package/node_modules/tardis-dev/dist/mappers/lighter.js.map +1 -1
  55. package/node_modules/tardis-dev/dist/mappers/mexc.d.ts +3 -3
  56. package/node_modules/tardis-dev/dist/mappers/mexcfutures.d.ts +3 -3
  57. package/node_modules/tardis-dev/dist/mappers/okex.d.ts +15 -15
  58. package/node_modules/tardis-dev/dist/mappers/phemex.d.ts +1 -1
  59. package/node_modules/tardis-dev/dist/mappers/polymarket.d.ts +3 -3
  60. package/node_modules/tardis-dev/dist/mappers/serum.d.ts +9 -9
  61. package/node_modules/tardis-dev/dist/mappers/upbit.d.ts +2 -2
  62. package/node_modules/tardis-dev/dist/parsemessage.d.ts +2 -0
  63. package/node_modules/tardis-dev/dist/parsemessage.d.ts.map +1 -0
  64. package/node_modules/tardis-dev/dist/parsemessage.js +31 -0
  65. package/node_modules/tardis-dev/dist/parsemessage.js.map +1 -0
  66. package/node_modules/tardis-dev/dist/realtimefeeds/bithumb.d.ts +9 -0
  67. package/node_modules/tardis-dev/dist/realtimefeeds/bithumb.d.ts.map +1 -0
  68. package/node_modules/tardis-dev/dist/realtimefeeds/bithumb.js +26 -0
  69. package/node_modules/tardis-dev/dist/realtimefeeds/bithumb.js.map +1 -0
  70. package/node_modules/tardis-dev/dist/realtimefeeds/bitvavo.d.ts +31 -0
  71. package/node_modules/tardis-dev/dist/realtimefeeds/bitvavo.d.ts.map +1 -0
  72. package/node_modules/tardis-dev/dist/realtimefeeds/bitvavo.js +85 -0
  73. package/node_modules/tardis-dev/dist/realtimefeeds/bitvavo.js.map +1 -0
  74. package/node_modules/tardis-dev/dist/realtimefeeds/index.d.ts.map +1 -1
  75. package/node_modules/tardis-dev/dist/realtimefeeds/index.js +7 -4
  76. package/node_modules/tardis-dev/dist/realtimefeeds/index.js.map +1 -1
  77. package/node_modules/tardis-dev/dist/realtimefeeds/lighter.d.ts +8 -2
  78. package/node_modules/tardis-dev/dist/realtimefeeds/lighter.d.ts.map +1 -1
  79. package/node_modules/tardis-dev/dist/realtimefeeds/lighter.js +8 -3
  80. package/node_modules/tardis-dev/dist/realtimefeeds/lighter.js.map +1 -1
  81. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.d.ts +1 -0
  82. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.d.ts.map +1 -1
  83. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.js +4 -5
  84. package/node_modules/tardis-dev/dist/realtimefeeds/realtimefeed.js.map +1 -1
  85. package/node_modules/tardis-dev/dist/replay.d.ts.map +1 -1
  86. package/node_modules/tardis-dev/dist/replay.js +5 -14
  87. package/node_modules/tardis-dev/dist/replay.js.map +1 -1
  88. package/node_modules/tardis-dev/dist/stream.d.ts.map +1 -1
  89. package/node_modules/tardis-dev/dist/stream.js +2 -1
  90. package/node_modules/tardis-dev/dist/stream.js.map +1 -1
  91. package/node_modules/tardis-dev/package.json +1 -1
  92. package/node_modules/tardis-dev/src/consts.ts +9 -0
  93. package/node_modules/tardis-dev/src/mappers/bithumb.ts +104 -0
  94. package/node_modules/tardis-dev/src/mappers/bitvavo.ts +256 -0
  95. package/node_modules/tardis-dev/src/mappers/index.ts +4 -0
  96. package/node_modules/tardis-dev/src/mappers/lighter.ts +69 -35
  97. package/node_modules/tardis-dev/src/parsemessage.ts +33 -0
  98. package/node_modules/tardis-dev/src/realtimefeeds/bithumb.ts +33 -0
  99. package/node_modules/tardis-dev/src/realtimefeeds/bitvavo.ts +107 -0
  100. package/node_modules/tardis-dev/src/realtimefeeds/index.ts +7 -4
  101. package/node_modules/tardis-dev/src/realtimefeeds/lighter.ts +10 -4
  102. package/node_modules/tardis-dev/src/realtimefeeds/realtimefeed.ts +4 -6
  103. package/node_modules/tardis-dev/src/replay.ts +5 -17
  104. package/node_modules/tardis-dev/src/stream.ts +3 -1
  105. package/package.json +2 -2
  106. package/src/ws/subscriptionsmappers.ts +20 -0
  107. package/node_modules/tardis-dev/dist/realtimefeeds/bitmex.d.ts +0 -11
  108. package/node_modules/tardis-dev/dist/realtimefeeds/bitmex.d.ts.map +0 -1
  109. package/node_modules/tardis-dev/dist/realtimefeeds/bitmex.js +0 -37
  110. package/node_modules/tardis-dev/dist/realtimefeeds/bitmex.js.map +0 -1
  111. package/node_modules/tardis-dev/src/realtimefeeds/bitmex.ts +0 -42
@@ -0,0 +1,33 @@
1
+ export function getMessageParser(exchange: string): (message: string) => any {
2
+ switch (exchange) {
3
+ case 'bitvavo':
4
+ return parseBitvavoMessage
5
+ case 'huobi-dm':
6
+ case 'huobi-dm-swap':
7
+ case 'huobi-dm-linear-swap':
8
+ case 'huobi-dm-options':
9
+ return parseHuobiMessage
10
+ case 'bithumb':
11
+ case 'upbit':
12
+ return parseSequentialIdMessage
13
+ default:
14
+ return JSON.parse
15
+ }
16
+ }
17
+
18
+ function parseBitvavoMessage(message: string) {
19
+ // Trades use timestampNs; book updates and getBook snapshots use timestamp.
20
+ // Quote only nanoseconds (19 digits), leaving millisecond trade timestamps numeric.
21
+ return JSON.parse(message.replace(/"(timestampNs|timestamp)":([0-9]{19})/g, '"$1":"$2"'))
22
+ }
23
+
24
+ function parseHuobiMessage(message: string) {
25
+ if (message.includes('.trade.detail')) {
26
+ message = message.replace(/"id":([0-9]+),/g, '"id":"$1",')
27
+ }
28
+ return JSON.parse(message)
29
+ }
30
+
31
+ function parseSequentialIdMessage(message: string) {
32
+ return JSON.parse(message.replace(/"sequential_id":([0-9]+),/g, '"sequential_id":"$1",'))
33
+ }
@@ -0,0 +1,33 @@
1
+ import { randomUUID } from 'node:crypto'
2
+ import { Filter } from '../types.ts'
3
+ import { RealTimeFeedBase } from './realtimefeed.ts'
4
+
5
+ export class BithumbRealTimeFeed extends RealTimeFeedBase {
6
+ protected readonly wssURL = 'wss://ws-api.bithumb.com/websocket/v1'
7
+ private readonly channels = new Set(['trade', 'orderbook', 'ticker'])
8
+
9
+ protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
10
+ const subscriptions = filters.map((filter) => {
11
+ if (!this.channels.has(filter.channel)) {
12
+ throw new Error(`BithumbRealTimeFeed unsupported channel ${filter.channel}`)
13
+ }
14
+
15
+ if (!filter.symbols || filter.symbols.length === 0) {
16
+ throw new Error('BithumbRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
17
+ }
18
+
19
+ const subscription = {
20
+ type: filter.channel,
21
+ codes: filter.symbols
22
+ }
23
+
24
+ return filter.channel === 'orderbook' ? { ...subscription, level: 1 } : subscription
25
+ })
26
+
27
+ return [[{ ticket: randomUUID() }, ...subscriptions, { format: 'DEFAULT' }]]
28
+ }
29
+
30
+ protected messageIsError(message: any): boolean {
31
+ return message.error !== undefined
32
+ }
33
+ }
@@ -0,0 +1,107 @@
1
+ import crypto from 'crypto'
2
+ import { Filter } from '../types.ts'
3
+ import { RealTimeFeedBase } from './realtimefeed.ts'
4
+
5
+ export class BitvavoRealTimeFeed extends RealTimeFeedBase {
6
+ protected readonly wssURL = 'wss://ws-mdpro.bitvavo.com/v2/'
7
+ protected readonly throttleSubscribeMS = 25
8
+ private nextRequestId = 1
9
+ private pendingAuthentication?: {
10
+ resolve: () => void
11
+ reject: (error: Error) => void
12
+ }
13
+
14
+ protected mapToSubscribeMessages(filters: Filter<string>[]) {
15
+ this.getCredentials()
16
+
17
+ const channels = filters
18
+ .filter((filter) => filter.channel !== 'getBook')
19
+ .map((filter) => {
20
+ if (!filter.symbols || filter.symbols.length === 0) {
21
+ throw new Error('BitvavoRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
22
+ }
23
+
24
+ return {
25
+ name: filter.channel === 'trade' ? 'trades' : filter.channel,
26
+ markets: filter.symbols
27
+ }
28
+ })
29
+
30
+ const snapshotRequests = (filters.find(({ channel }) => channel === 'getBook')?.symbols ?? []).map((symbol) => ({
31
+ action: 'getBook',
32
+ requestId: this.nextRequestId++,
33
+ market: symbol,
34
+ depth: 1000
35
+ }))
36
+
37
+ return [...(channels.length === 0 ? [] : [{ action: 'subscribe', channels }]), ...snapshotRequests]
38
+ }
39
+
40
+ protected async onConnected() {
41
+ const { key, secret } = this.getCredentials()
42
+ const timestamp = Date.now()
43
+ const signature = crypto.createHmac('sha256', secret).update(`${timestamp}GET/v2/websocket`).digest('hex')
44
+
45
+ let resolveAuthentication!: () => void
46
+ let rejectAuthentication!: (error: Error) => void
47
+ const authenticationResponse = new Promise<void>((resolve, reject) => {
48
+ resolveAuthentication = resolve
49
+ rejectAuthentication = reject
50
+ })
51
+ const pendingAuthentication = {
52
+ resolve: resolveAuthentication,
53
+ reject: rejectAuthentication
54
+ }
55
+ this.pendingAuthentication = pendingAuthentication
56
+
57
+ const timeout = setTimeout(() => {
58
+ pendingAuthentication.reject(new Error('Bitvavo Market Data Pro authentication timed out'))
59
+ }, 10000)
60
+
61
+ this.send({ action: 'authenticate', key, signature, timestamp })
62
+
63
+ try {
64
+ await authenticationResponse
65
+ } finally {
66
+ clearTimeout(timeout)
67
+ if (this.pendingAuthentication === pendingAuthentication) {
68
+ this.pendingAuthentication = undefined
69
+ }
70
+ }
71
+ }
72
+
73
+ protected onMessage(message: BitvavoRealtimeMessage) {
74
+ if (message.event !== 'authenticate') {
75
+ return
76
+ }
77
+
78
+ if (message.authenticated === true) {
79
+ this.pendingAuthentication?.resolve()
80
+ } else {
81
+ this.pendingAuthentication?.reject(new Error('Bitvavo Market Data Pro authentication failed'))
82
+ }
83
+ }
84
+
85
+ protected messageIsError(message: BitvavoRealtimeMessage) {
86
+ const isError = message.event === 'error' || message.error !== undefined
87
+ if (isError) {
88
+ this.pendingAuthentication?.reject(new Error(`Bitvavo Market Data Pro error: ${JSON.stringify(message)}`))
89
+ }
90
+ return isError
91
+ }
92
+
93
+ private getCredentials() {
94
+ const key = process.env.BITVAVO_API_KEY
95
+ const secret = process.env.BITVAVO_API_SECRET
96
+ if (key === undefined || secret === undefined) {
97
+ throw new Error('BitvavoRealTimeFeed requires BITVAVO_API_KEY and BITVAVO_API_SECRET environment variables')
98
+ }
99
+ return { key, secret }
100
+ }
101
+ }
102
+
103
+ type BitvavoRealtimeMessage = {
104
+ event?: string
105
+ authenticated?: boolean
106
+ error?: unknown
107
+ }
@@ -10,8 +10,9 @@ import {
10
10
  import { BinanceDexRealTimeFeed } from './binancedex.ts'
11
11
  import { BitfinexRealTimeFeed } from './bitfinex.ts'
12
12
  import { BitflyerRealTimeFeed } from './bitflyer.ts'
13
- import { BitmexRealTimeFeed } from './bitmex.ts'
13
+ import { BithumbRealTimeFeed } from './bithumb.ts'
14
14
  import { BitstampRealTimeFeed } from './bitstamp.ts'
15
+ import { BitvavoRealTimeFeed } from './bitvavo.ts'
15
16
  import { BybitOptionsRealTimeDataFeed, BybitRealTimeDataFeed, BybitSpotRealTimeDataFeed } from './bybit.ts'
16
17
  import { CoinbaseRealTimeFeed } from './coinbase.ts'
17
18
  import { CryptofacilitiesRealTimeFeed } from './cryptofacilities.ts'
@@ -52,7 +53,7 @@ import { DydxV4RealTimeFeed } from './dydx_v4.ts'
52
53
  import { BitgetFuturesRealTimeFeed, BitgetRealTimeFeed } from './bitget.ts'
53
54
  import { CoinbaseInternationalRealTimeFeed } from './coinbaseinternational.ts'
54
55
  import { HyperliquidRealTimeFeed } from './hyperliquid.ts'
55
- import { LighterRealTimeFeed } from './lighter.ts'
56
+ import { LighterRealTimeFeed, LighterRhRealTimeFeed } from './lighter.ts'
56
57
  import { BullishRealTimeFeed } from './bullish.ts'
57
58
  import { MexcFuturesRealTimeFeed } from './mexcfutures.ts'
58
59
  import { MexcRealTimeFeed } from './mexc.ts'
@@ -65,7 +66,6 @@ const realTimeFeedsMap: {
65
66
  } = {
66
67
  aster: AsterRealTimeFeed,
67
68
  'aster-futures': AsterFuturesRealTimeFeed,
68
- bitmex: BitmexRealTimeFeed,
69
69
  binance: BinanceRealTimeFeed,
70
70
  'binance-jersey': BinanceJerseyRealTimeFeed,
71
71
  'binance-us': BinanceUSRealTimeFeed,
@@ -75,7 +75,9 @@ const realTimeFeedsMap: {
75
75
  bitfinex: BitfinexRealTimeFeed,
76
76
  'bitfinex-derivatives': BitfinexRealTimeFeed,
77
77
  bitflyer: BitflyerRealTimeFeed,
78
+ bithumb: BithumbRealTimeFeed,
78
79
  bitstamp: BitstampRealTimeFeed,
80
+ bitvavo: BitvavoRealTimeFeed,
79
81
  coinbase: CoinbaseRealTimeFeed,
80
82
  cryptofacilities: CryptofacilitiesRealTimeFeed,
81
83
  deribit: DeribitRealTimeDataFeed,
@@ -123,6 +125,7 @@ const realTimeFeedsMap: {
123
125
  'coinbase-international': CoinbaseInternationalRealTimeFeed,
124
126
  hyperliquid: HyperliquidRealTimeFeed,
125
127
  lighter: LighterRealTimeFeed,
128
+ 'lighter-rh': LighterRhRealTimeFeed,
126
129
  bullish: BullishRealTimeFeed,
127
130
  mexc: MexcRealTimeFeed,
128
131
  polymarket: PolymarketRealTimeFeed
@@ -133,7 +136,7 @@ export function getRealTimeFeedFactory(exchange: Exchange): RealTimeFeed {
133
136
  return realTimeFeedsMap[exchange]!
134
137
  }
135
138
 
136
- throw new Error(`not supported exchange ${exchange}`)
139
+ throw new Error(`Real-time streaming is not supported for exchange ${exchange}`)
137
140
  }
138
141
 
139
142
  export function createRealTimeFeed(
@@ -1,9 +1,7 @@
1
1
  import { Filter } from '../types.ts'
2
2
  import { RealTimeFeedBase } from './realtimefeed.ts'
3
3
 
4
- export class LighterRealTimeFeed extends RealTimeFeedBase {
5
- protected wssURL = 'wss://mainnet.zklighter.elliot.ai/stream'
6
-
4
+ abstract class RealTimeFeed extends RealTimeFeedBase {
7
5
  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
8
6
  return filters.flatMap((filter) => {
9
7
  if (filter.channel === 'market_stats') {
@@ -15,7 +13,7 @@ export class LighterRealTimeFeed extends RealTimeFeedBase {
15
13
  }
16
14
 
17
15
  if (!filter.symbols || filter.symbols.length === 0) {
18
- throw new Error('LighterRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
16
+ throw new Error(`${this._exchange} RealTimeFeed requires explicitly specified symbols when subscribing to live feed`)
19
17
  }
20
18
 
21
19
  return filter.symbols.map((marketId) => ({
@@ -29,3 +27,11 @@ export class LighterRealTimeFeed extends RealTimeFeedBase {
29
27
  return message.error !== undefined
30
28
  }
31
29
  }
30
+
31
+ export class LighterRealTimeFeed extends RealTimeFeed {
32
+ protected wssURL = 'wss://mainnet.zklighter.elliot.ai/stream'
33
+ }
34
+
35
+ export class LighterRhRealTimeFeed extends RealTimeFeed {
36
+ protected wssURL = 'wss://api.rh.lighter.xyz/stream'
37
+ }
@@ -4,6 +4,7 @@ import { PassThrough, Writable } from 'stream'
4
4
  import { setTimeout as sleep } from 'node:timers/promises'
5
5
  import { createDebug, type DebugLogger } from '../debug.ts'
6
6
  import { getProxyAgent, HttpClientError, ONE_SEC_IN_MS, optimizeFilters } from '../handy.ts'
7
+ import { getMessageParser } from '../parsemessage.ts'
7
8
  import { createManagedRealTimeIterator, mergeRealTime, type ManagedRealTimeIterator } from '../realtimeiterator.ts'
8
9
  import { Exchange, Filter } from '../types.ts'
9
10
 
@@ -39,6 +40,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
39
40
  protected readonly throttleSubscribeMS: number = 0
40
41
  protected readonly manualSnapshotsBuffer: any[] = []
41
42
  private readonly _filters: Filter<string>[]
43
+ private readonly _parseJson: (message: string) => any
42
44
  private _wsClientOptions: WebSocket.ClientOptions | ClientRequestArgs
43
45
  private _closed = false
44
46
  private readonly _closeController = new AbortController()
@@ -53,6 +55,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
53
55
  private readonly _onError?: (error: Error) => void
54
56
  ) {
55
57
  this._filters = optimizeFilters(filters)
58
+ this._parseJson = getMessageParser(_exchange)
56
59
  this.debug = createDebug(`tardis-dev:realtime:${_exchange}`)
57
60
 
58
61
  this._wsClientOptions = {
@@ -149,11 +152,6 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
149
152
  message = this.decompress(message)
150
153
  }
151
154
 
152
- // hack to handle huobi long numeric id for trades
153
- if (this._exchange.startsWith('huobi-') && message.includes('.trade.detail')) {
154
- message = message.toString().replace(/"id":([0-9]+),/g, '"id":"$1",') as any
155
- }
156
-
157
155
  const messageDeserialized = this.parseMessage(message)
158
156
 
159
157
  if (this.messageIsError(messageDeserialized)) {
@@ -270,7 +268,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
270
268
  protected abstract messageIsError(message: any): boolean
271
269
 
272
270
  protected parseMessage(message: Buffer<ArrayBufferLike>): any {
273
- return JSON.parse(message as any)
271
+ return this._parseJson(message.toString())
274
272
  }
275
273
 
276
274
  protected sendCustomPing: (() => void) | undefined = undefined
@@ -10,6 +10,7 @@ import { debug } from './debug.ts'
10
10
  import { addDays, createNormalizedSymbolFilter, getFilters, parseAsUTCDate, wait } from './handy.ts'
11
11
  import { Mapper, MapperFactory, normalizeBookChanges } from './mappers/index.ts'
12
12
  import { getOptions } from './options.ts'
13
+ import { getMessageParser } from './parsemessage.ts'
13
14
  import { Disconnect, Exchange, FilterForExchange } from './types.ts'
14
15
  import { WorkerJobPayload, WorkerMessage, WorkerSignal } from './worker.ts'
15
16
  import { createZstdDecompressionStream } from './zstd.ts'
@@ -53,6 +54,7 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
53
54
  : { localTimestamp: Date; message: any }
54
55
  > {
55
56
  let lastMessageWasUndefined = false
57
+ const parseMessage = getMessageParser(exchange)
56
58
 
57
59
  const lineBatches = replayLineBatches({ exchange, from, to, filters, apiKey, autoCleanup, waitWhenDataNotYetAvailable })
58
60
  for await (const bufferLines of lineBatches) {
@@ -69,7 +71,7 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
69
71
  message: bufferLine.slice(DATE_MESSAGE_SPLIT_INDEX + 1)
70
72
  })
71
73
  } else {
72
- const message = parseReplayMessage(exchange, bufferLine)
74
+ const message = parseMessage(bufferLine.toString('utf8', DATE_MESSAGE_SPLIT_INDEX + 1))
73
75
  const localTimestamp = parseReplayTimestamp(bufferLine)
74
76
  if (withMicroseconds) {
75
77
  localTimestamp.μs = parseReplayMicroseconds(bufferLine)
@@ -244,21 +246,6 @@ async function* replayLineBatches<T extends Exchange>({
244
246
  }
245
247
  }
246
248
 
247
- function parseReplayMessage(exchange: Exchange, bufferLine: Buffer) {
248
- let messageString = bufferLine.toString('utf8', DATE_MESSAGE_SPLIT_INDEX + 1)
249
-
250
- // hack to handle huobi long numeric id for trades
251
- if (exchange.startsWith('huobi-') && messageString.includes('.trade.detail')) {
252
- messageString = messageString.replace(/"id":([0-9]+),/g, '"id":"$1",')
253
- }
254
- // hack to handle upbit long numeric id for trades
255
- if (exchange === 'upbit' && messageString.includes('sequential_id')) {
256
- messageString = messageString.replace(/"sequential_id":([0-9]+),/g, '"sequential_id":"$1",')
257
- }
258
-
259
- return JSON.parse(messageString)
260
- }
261
-
262
249
  async function cleanupSlice(slicePath: string) {
263
250
  try {
264
251
  await rm(slicePath, { force: true })
@@ -394,6 +381,7 @@ async function* normalizeReplayLineBatches(
394
381
  filter?: (symbol: string) => boolean
395
382
  ) {
396
383
  // This intentionally keeps mapper calls lazy. Custom normalizers must not process later raw messages before the consumer asks for them.
384
+ const parseMessage = getMessageParser(exchange)
397
385
  let previousLocalTimestamp: Date | undefined
398
386
  let activeMappers: Mapper<any, any>[] | undefined = initialMappers
399
387
  if (activeMappers.length === 0) {
@@ -410,7 +398,7 @@ async function* normalizeReplayLineBatches(
410
398
  if (bufferLine.length === 0) {
411
399
  decodedMessages.push(undefined)
412
400
  } else {
413
- const message = parseReplayMessage(exchange, bufferLine)
401
+ const message = parseMessage(bufferLine.toString('utf8', DATE_MESSAGE_SPLIT_INDEX + 1))
414
402
  const localTimestamp = parseReplayTimestamp(bufferLine)
415
403
  localTimestamp.μs = parseReplayMicroseconds(bufferLine)
416
404
  decodedMessages.push({ localTimestamp, message })
@@ -2,7 +2,7 @@ import { debug } from './debug.ts'
2
2
  import { createNormalizedSymbolFilter, getFilters, normalizeMessages } from './handy.ts'
3
3
  import { MapperFactory } from './mappers/index.ts'
4
4
  import { createManagedRealTimeIterator, type ManagedRealTimeIterator } from './realtimeiterator.ts'
5
- import { createRealTimeFeed } from './realtimefeeds/index.ts'
5
+ import { createRealTimeFeed, getRealTimeFeedFactory } from './realtimefeeds/index.ts'
6
6
  import { Disconnect, Exchange, Filter, FilterForExchange } from './types.ts'
7
7
 
8
8
  type MapperOutput<T> = T extends MapperFactory<any, infer U> ? U : never
@@ -82,6 +82,8 @@ async function* _streamNormalized<T extends Exchange, U extends MapperFactory<T,
82
82
  state: NormalizedStreamState,
83
83
  ...normalizers: U
84
84
  ): AsyncIterableIterator<StreamNormalizedMessage<U, Z>> {
85
+ getRealTimeFeedFactory(exchange)
86
+
85
87
  while (state.closed === false) {
86
88
  try {
87
89
  const createMappers = (localTimestamp: Date) => normalizers.map((m) => m(exchange, localTimestamp))
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tardis-machine",
3
- "version": "18.2.1",
3
+ "version": "18.3.0",
4
4
  "engines": {
5
5
  "node": ">=24.19"
6
6
  },
@@ -76,7 +76,7 @@
76
76
  "license": "MPL-2.0",
77
77
  "dependencies": {
78
78
  "debug": "^4.4.3",
79
- "tardis-dev": "^18.2.2",
79
+ "tardis-dev": "^18.3.0",
80
80
  "ws": "^8.21.3"
81
81
  },
82
82
  "bundleDependencies": true,
@@ -745,6 +745,23 @@ const lighterMapper: SubscriptionMapper = {
745
745
  }
746
746
  }
747
747
 
748
+ const bitvavoMapper: SubscriptionMapper = {
749
+ canHandle: (message: any) => {
750
+ return message.action === 'subscribe' || message.action === 'getBook'
751
+ },
752
+
753
+ map: (message: any) => {
754
+ if (message.action === 'getBook') {
755
+ return [{ channel: 'getBook', symbols: [message.market] }]
756
+ }
757
+
758
+ return message.channels.map((channel: any) => ({
759
+ channel: channel.name === 'trades' ? 'trade' : channel.name,
760
+ symbols: channel.markets
761
+ }))
762
+ }
763
+ }
764
+
748
765
  const bullishMapper: SubscriptionMapper = {
749
766
  canHandle: (message: any) => {
750
767
  return message.method === 'subscribe'
@@ -834,6 +851,8 @@ export const subscriptionsMappers: Record<Exchange, SubscriptionMapper> = {
834
851
  'dydx-v4': dydxMapper,
835
852
  'huobi-dm-options': huobiMapper,
836
853
  upbit: upbitMapper,
854
+ bithumb: upbitMapper,
855
+ bitvavo: bitvavoMapper,
837
856
  serum: serumMaper,
838
857
  'star-atlas': serumMaper,
839
858
  mango: serumMaper,
@@ -854,6 +873,7 @@ export const subscriptionsMappers: Record<Exchange, SubscriptionMapper> = {
854
873
  'coinbase-international': coinbaseInternationalMapper,
855
874
  hyperliquid: hyperliquidMapper,
856
875
  lighter: lighterMapper,
876
+ 'lighter-rh': lighterMapper,
857
877
  bullish: bullishMapper,
858
878
  polymarket: polymarketMapper
859
879
  }
@@ -1,11 +0,0 @@
1
- import { Filter } from '../types.ts';
2
- import { RealTimeFeedBase } from './realtimefeed.ts';
3
- export declare class BitmexRealTimeFeed extends RealTimeFeedBase {
4
- protected readonly wssURL = "wss://ws.bitmex.com/realtime";
5
- protected mapToSubscribeMessages(filters: Filter<string>[]): {
6
- op: string;
7
- args: string[];
8
- }[];
9
- protected messageIsError(message: any): boolean;
10
- }
11
- //# sourceMappingURL=bitmex.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"bitmex.d.ts","sourceRoot":"","sources":["../../src/realtimefeeds/bitmex.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAEpD,qBAAa,kBAAmB,SAAQ,gBAAgB;IACtD,SAAS,CAAC,QAAQ,CAAC,MAAM,kCAAiC;IAE1D,SAAS,CAAC,sBAAsB,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE;;;;IAwB1D,SAAS,CAAC,cAAc,CAAC,OAAO,EAAE,GAAG,GAAG,OAAO;CAUhD"}
@@ -1,37 +0,0 @@
1
- import { batch } from "../handy.js";
2
- import { RealTimeFeedBase } from "./realtimefeed.js";
3
- export class BitmexRealTimeFeed extends RealTimeFeedBase {
4
- wssURL = 'wss://ws.bitmex.com/realtime';
5
- mapToSubscribeMessages(filters) {
6
- return filters
7
- .map((filter) => {
8
- if (!filter.symbols || filter.symbols.length === 0) {
9
- return [
10
- {
11
- op: 'subscribe',
12
- args: [filter.channel]
13
- }
14
- ];
15
- }
16
- const subscribeMessages = [];
17
- for (const symbolsBatch of batch(filter.symbols, 10)) {
18
- subscribeMessages.push({
19
- op: 'subscribe',
20
- args: symbolsBatch.map((s) => `${filter.channel}:${s}`)
21
- });
22
- }
23
- return subscribeMessages;
24
- })
25
- .flatMap((s) => s);
26
- }
27
- messageIsError(message) {
28
- if (message.error !== undefined) {
29
- return true;
30
- }
31
- if ('subscribe' in message && message.success === false) {
32
- return true;
33
- }
34
- return false;
35
- }
36
- }
37
- //# sourceMappingURL=bitmex.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"bitmex.js","sourceRoot":"","sources":["../../src/realtimefeeds/bitmex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA;AAEnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAEpD,MAAM,OAAO,kBAAmB,SAAQ,gBAAgB;IACnC,MAAM,GAAG,8BAA8B,CAAA;IAEhD,sBAAsB,CAAC,OAAyB;QACxD,OAAO,OAAO;aACX,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACnD,OAAO;oBACL;wBACE,EAAE,EAAE,WAAW;wBACf,IAAI,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC;qBACvB;iBACF,CAAA;YACH,CAAC;YACD,MAAM,iBAAiB,GAAG,EAAE,CAAA;YAC5B,KAAK,MAAM,YAAY,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC;gBACrD,iBAAiB,CAAC,IAAI,CAAC;oBACrB,EAAE,EAAE,WAAW;oBACf,IAAI,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;iBACxD,CAAC,CAAA;YACJ,CAAC;YAED,OAAO,iBAAiB,CAAA;QAC1B,CAAC,CAAC;aACD,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;IACtB,CAAC;IAES,cAAc,CAAC,OAAY;QACnC,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,OAAO,IAAI,CAAA;QACb,CAAC;QAED,IAAI,WAAW,IAAI,OAAO,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YACxD,OAAO,IAAI,CAAA;QACb,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;CACF"}
@@ -1,42 +0,0 @@
1
- import { batch } from '../handy.ts'
2
- import { Filter } from '../types.ts'
3
- import { RealTimeFeedBase } from './realtimefeed.ts'
4
-
5
- export class BitmexRealTimeFeed extends RealTimeFeedBase {
6
- protected readonly wssURL = 'wss://ws.bitmex.com/realtime'
7
-
8
- protected mapToSubscribeMessages(filters: Filter<string>[]) {
9
- return filters
10
- .map((filter) => {
11
- if (!filter.symbols || filter.symbols.length === 0) {
12
- return [
13
- {
14
- op: 'subscribe',
15
- args: [filter.channel]
16
- }
17
- ]
18
- }
19
- const subscribeMessages = []
20
- for (const symbolsBatch of batch(filter.symbols, 10)) {
21
- subscribeMessages.push({
22
- op: 'subscribe',
23
- args: symbolsBatch.map((s) => `${filter.channel}:${s}`)
24
- })
25
- }
26
-
27
- return subscribeMessages
28
- })
29
- .flatMap((s) => s)
30
- }
31
-
32
- protected messageIsError(message: any): boolean {
33
- if (message.error !== undefined) {
34
- return true
35
- }
36
-
37
- if ('subscribe' in message && message.success === false) {
38
- return true
39
- }
40
- return false
41
- }
42
- }