tardis-dev 18.10.2 → 18.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. package/README.md +2 -0
  2. package/dist/mappers/aster.js +0 -3
  3. package/dist/mappers/aster.js.map +1 -1
  4. package/dist/mappers/backpack.d.ts.map +1 -1
  5. package/dist/mappers/backpack.js +9 -6
  6. package/dist/mappers/backpack.js.map +1 -1
  7. package/dist/mappers/bitvavo.js +9 -8
  8. package/dist/mappers/bitvavo.js.map +1 -1
  9. package/dist/mappers/deribit.js +4 -4
  10. package/dist/mappers/deribit.js.map +1 -1
  11. package/dist/realtimefeeds/aster.d.ts +3 -2
  12. package/dist/realtimefeeds/aster.d.ts.map +1 -1
  13. package/dist/realtimefeeds/aster.js +12 -2
  14. package/dist/realtimefeeds/aster.js.map +1 -1
  15. package/dist/realtimefeeds/backpack.d.ts +3 -2
  16. package/dist/realtimefeeds/backpack.d.ts.map +1 -1
  17. package/dist/realtimefeeds/backpack.js +9 -2
  18. package/dist/realtimefeeds/backpack.js.map +1 -1
  19. package/dist/realtimefeeds/binance.d.ts +7 -5
  20. package/dist/realtimefeeds/binance.d.ts.map +1 -1
  21. package/dist/realtimefeeds/binance.js +42 -9
  22. package/dist/realtimefeeds/binance.js.map +1 -1
  23. package/dist/realtimefeeds/binanceeuropeanoptions.d.ts +2 -2
  24. package/dist/realtimefeeds/binanceeuropeanoptions.d.ts.map +1 -1
  25. package/dist/realtimefeeds/binanceeuropeanoptions.js +5 -5
  26. package/dist/realtimefeeds/binanceeuropeanoptions.js.map +1 -1
  27. package/dist/realtimefeeds/bitget.d.ts +2 -1
  28. package/dist/realtimefeeds/bitget.d.ts.map +1 -1
  29. package/dist/realtimefeeds/bitget.js +10 -0
  30. package/dist/realtimefeeds/bitget.js.map +1 -1
  31. package/dist/realtimefeeds/bitstamp.d.ts +1 -1
  32. package/dist/realtimefeeds/bitstamp.d.ts.map +1 -1
  33. package/dist/realtimefeeds/bitstamp.js +2 -2
  34. package/dist/realtimefeeds/bitstamp.js.map +1 -1
  35. package/dist/realtimefeeds/bitvavo.d.ts +2 -1
  36. package/dist/realtimefeeds/bitvavo.d.ts.map +1 -1
  37. package/dist/realtimefeeds/bitvavo.js +10 -0
  38. package/dist/realtimefeeds/bitvavo.js.map +1 -1
  39. package/dist/realtimefeeds/bullish.d.ts +2 -2
  40. package/dist/realtimefeeds/bullish.d.ts.map +1 -1
  41. package/dist/realtimefeeds/bullish.js +4 -4
  42. package/dist/realtimefeeds/bullish.js.map +1 -1
  43. package/dist/realtimefeeds/bybit.d.ts +3 -2
  44. package/dist/realtimefeeds/bybit.d.ts.map +1 -1
  45. package/dist/realtimefeeds/bybit.js +18 -3
  46. package/dist/realtimefeeds/bybit.js.map +1 -1
  47. package/dist/realtimefeeds/coinbase.d.ts +4 -1
  48. package/dist/realtimefeeds/coinbase.d.ts.map +1 -1
  49. package/dist/realtimefeeds/coinbase.js +22 -0
  50. package/dist/realtimefeeds/coinbase.js.map +1 -1
  51. package/dist/realtimefeeds/coinbasederivatives.d.ts +3 -7
  52. package/dist/realtimefeeds/coinbasederivatives.d.ts.map +1 -1
  53. package/dist/realtimefeeds/coinbasederivatives.js +10 -12
  54. package/dist/realtimefeeds/coinbasederivatives.js.map +1 -1
  55. package/dist/realtimefeeds/cryptofacilities.d.ts +2 -1
  56. package/dist/realtimefeeds/cryptofacilities.d.ts.map +1 -1
  57. package/dist/realtimefeeds/cryptofacilities.js +6 -0
  58. package/dist/realtimefeeds/cryptofacilities.js.map +1 -1
  59. package/dist/realtimefeeds/delta.d.ts +2 -1
  60. package/dist/realtimefeeds/delta.d.ts.map +1 -1
  61. package/dist/realtimefeeds/delta.js +6 -0
  62. package/dist/realtimefeeds/delta.js.map +1 -1
  63. package/dist/realtimefeeds/delta_india.d.ts +4 -3
  64. package/dist/realtimefeeds/delta_india.d.ts.map +1 -1
  65. package/dist/realtimefeeds/delta_india.js +22 -17
  66. package/dist/realtimefeeds/delta_india.js.map +1 -1
  67. package/dist/realtimefeeds/deribit.d.ts +2 -1
  68. package/dist/realtimefeeds/deribit.d.ts.map +1 -1
  69. package/dist/realtimefeeds/deribit.js +11 -0
  70. package/dist/realtimefeeds/deribit.js.map +1 -1
  71. package/dist/realtimefeeds/ftx.d.ts +2 -2
  72. package/dist/realtimefeeds/ftx.d.ts.map +1 -1
  73. package/dist/realtimefeeds/ftx.js +4 -4
  74. package/dist/realtimefeeds/ftx.js.map +1 -1
  75. package/dist/realtimefeeds/gateio.d.ts +2 -1
  76. package/dist/realtimefeeds/gateio.d.ts.map +1 -1
  77. package/dist/realtimefeeds/gateio.js +10 -0
  78. package/dist/realtimefeeds/gateio.js.map +1 -1
  79. package/dist/realtimefeeds/gateiofutures.d.ts +5 -4
  80. package/dist/realtimefeeds/gateiofutures.d.ts.map +1 -1
  81. package/dist/realtimefeeds/gateiofutures.js +24 -14
  82. package/dist/realtimefeeds/gateiofutures.js.map +1 -1
  83. package/dist/realtimefeeds/gemini.d.ts +4 -1
  84. package/dist/realtimefeeds/gemini.d.ts.map +1 -1
  85. package/dist/realtimefeeds/gemini.js +25 -0
  86. package/dist/realtimefeeds/gemini.js.map +1 -1
  87. package/dist/realtimefeeds/hitbtc.d.ts +2 -1
  88. package/dist/realtimefeeds/hitbtc.d.ts.map +1 -1
  89. package/dist/realtimefeeds/hitbtc.js +10 -0
  90. package/dist/realtimefeeds/hitbtc.js.map +1 -1
  91. package/dist/realtimefeeds/huobi.d.ts +5 -4
  92. package/dist/realtimefeeds/huobi.d.ts.map +1 -1
  93. package/dist/realtimefeeds/huobi.js +28 -9
  94. package/dist/realtimefeeds/huobi.js.map +1 -1
  95. package/dist/realtimefeeds/index.d.ts +1 -1
  96. package/dist/realtimefeeds/index.d.ts.map +1 -1
  97. package/dist/realtimefeeds/index.js +2 -2
  98. package/dist/realtimefeeds/index.js.map +1 -1
  99. package/dist/realtimefeeds/kucoin.d.ts +2 -1
  100. package/dist/realtimefeeds/kucoin.d.ts.map +1 -1
  101. package/dist/realtimefeeds/kucoin.js +10 -0
  102. package/dist/realtimefeeds/kucoin.js.map +1 -1
  103. package/dist/realtimefeeds/kucoinfutures.d.ts +4 -3
  104. package/dist/realtimefeeds/kucoinfutures.d.ts.map +1 -1
  105. package/dist/realtimefeeds/kucoinfutures.js +15 -4
  106. package/dist/realtimefeeds/kucoinfutures.js.map +1 -1
  107. package/dist/realtimefeeds/lighter.d.ts +2 -1
  108. package/dist/realtimefeeds/lighter.d.ts.map +1 -1
  109. package/dist/realtimefeeds/lighter.js +10 -0
  110. package/dist/realtimefeeds/lighter.js.map +1 -1
  111. package/dist/realtimefeeds/mexc.d.ts +2 -1
  112. package/dist/realtimefeeds/mexc.d.ts.map +1 -1
  113. package/dist/realtimefeeds/mexc.js +10 -0
  114. package/dist/realtimefeeds/mexc.js.map +1 -1
  115. package/dist/realtimefeeds/mexcfutures.d.ts +2 -1
  116. package/dist/realtimefeeds/mexcfutures.d.ts.map +1 -1
  117. package/dist/realtimefeeds/mexcfutures.js +11 -0
  118. package/dist/realtimefeeds/mexcfutures.js.map +1 -1
  119. package/dist/realtimefeeds/okex.d.ts +7 -5
  120. package/dist/realtimefeeds/okex.d.ts.map +1 -1
  121. package/dist/realtimefeeds/okex.js +32 -11
  122. package/dist/realtimefeeds/okex.js.map +1 -1
  123. package/dist/realtimefeeds/poloniex.d.ts +2 -1
  124. package/dist/realtimefeeds/poloniex.d.ts.map +1 -1
  125. package/dist/realtimefeeds/poloniex.js +11 -0
  126. package/dist/realtimefeeds/poloniex.js.map +1 -1
  127. package/dist/realtimefeeds/polymarket.d.ts +1 -1
  128. package/dist/realtimefeeds/polymarket.d.ts.map +1 -1
  129. package/dist/realtimefeeds/polymarket.js +3 -3
  130. package/dist/realtimefeeds/polymarket.js.map +1 -1
  131. package/dist/realtimefeeds/realtimefeed.d.ts +20 -4
  132. package/dist/realtimefeeds/realtimefeed.d.ts.map +1 -1
  133. package/dist/realtimefeeds/realtimefeed.js +41 -3
  134. package/dist/realtimefeeds/realtimefeed.js.map +1 -1
  135. package/dist/realtimefeeds/whitebit.d.ts +4 -11
  136. package/dist/realtimefeeds/whitebit.d.ts.map +1 -1
  137. package/dist/realtimefeeds/whitebit.js +14 -15
  138. package/dist/realtimefeeds/whitebit.js.map +1 -1
  139. package/dist/realtimefeeds/woox.d.ts +2 -1
  140. package/dist/realtimefeeds/woox.d.ts.map +1 -1
  141. package/dist/realtimefeeds/woox.js +11 -0
  142. package/dist/realtimefeeds/woox.js.map +1 -1
  143. package/dist/stream.d.ts +4 -2
  144. package/dist/stream.d.ts.map +1 -1
  145. package/dist/stream.js +10 -9
  146. package/dist/stream.js.map +1 -1
  147. package/package.json +1 -1
  148. package/src/mappers/aster.ts +0 -6
  149. package/src/mappers/backpack.ts +10 -4
  150. package/src/mappers/bitvavo.ts +18 -16
  151. package/src/mappers/deribit.ts +4 -4
  152. package/src/realtimefeeds/aster.ts +21 -3
  153. package/src/realtimefeeds/backpack.ts +17 -3
  154. package/src/realtimefeeds/binance.ts +68 -11
  155. package/src/realtimefeeds/binanceeuropeanoptions.ts +14 -5
  156. package/src/realtimefeeds/bitget.ts +13 -1
  157. package/src/realtimefeeds/bitstamp.ts +8 -2
  158. package/src/realtimefeeds/bitvavo.ts +13 -1
  159. package/src/realtimefeeds/bullish.ts +11 -4
  160. package/src/realtimefeeds/bybit.ts +27 -4
  161. package/src/realtimefeeds/coinbase.ts +34 -1
  162. package/src/realtimefeeds/coinbasederivatives.ts +18 -13
  163. package/src/realtimefeeds/cryptofacilities.ts +9 -1
  164. package/src/realtimefeeds/delta.ts +9 -1
  165. package/src/realtimefeeds/delta_india.ts +31 -17
  166. package/src/realtimefeeds/deribit.ts +13 -1
  167. package/src/realtimefeeds/ftx.ts +11 -4
  168. package/src/realtimefeeds/gateio.ts +13 -1
  169. package/src/realtimefeeds/gateiofutures.ts +37 -17
  170. package/src/realtimefeeds/gemini.ts +28 -1
  171. package/src/realtimefeeds/hitbtc.ts +13 -1
  172. package/src/realtimefeeds/huobi.ts +65 -10
  173. package/src/realtimefeeds/index.ts +3 -2
  174. package/src/realtimefeeds/kucoin.ts +12 -1
  175. package/src/realtimefeeds/kucoinfutures.ts +24 -5
  176. package/src/realtimefeeds/lighter.ts +12 -1
  177. package/src/realtimefeeds/mexc.ts +12 -1
  178. package/src/realtimefeeds/mexcfutures.ts +13 -1
  179. package/src/realtimefeeds/okex.ts +76 -11
  180. package/src/realtimefeeds/poloniex.ts +13 -1
  181. package/src/realtimefeeds/polymarket.ts +4 -3
  182. package/src/realtimefeeds/realtimefeed.ts +53 -5
  183. package/src/realtimefeeds/whitebit.ts +21 -14
  184. package/src/realtimefeeds/woox.ts +13 -1
  185. package/src/stream.ts +37 -8
@@ -1,7 +1,7 @@
1
1
  import { Writable } from 'stream'
2
2
  import { getJSON, getRandomString, ONE_SEC_IN_MS, postJSON, wait } from '../handy.ts'
3
3
  import { Filter } from '../types.ts'
4
- import { MultiConnectionRealTimeFeedBase, PoolingClientBase, RealTimeFeedBase } from './realtimefeed.ts'
4
+ import { MultiConnectionRealTimeFeedBase, PoolingClientBase, RealTimeFeedBase, type Sequence } from './realtimefeed.ts'
5
5
 
6
6
  const kucoinHttpOptions = {
7
7
  timeout: 10 * 1000,
@@ -15,11 +15,17 @@ const kucoinHttpOptions = {
15
15
  export class KucoinFuturesRealTimeFeed extends MultiConnectionRealTimeFeedBase {
16
16
  private _httpURL = 'https://api-futures.kucoin.com/api'
17
17
 
18
- protected *_getRealTimeFeeds(exchange: string, filters: Filter<string>[], timeoutIntervalMS?: number, onError?: (error: Error) => void) {
18
+ protected *_getRealTimeFeeds(
19
+ exchange: string,
20
+ filters: Filter<string>[],
21
+ timeoutIntervalMS?: number,
22
+ onError?: (error: Error) => void,
23
+ validateSequence?: boolean
24
+ ) {
19
25
  const wsFilters = filters.filter((f) => f.channel !== 'contract/details')
20
26
 
21
27
  if (wsFilters.length > 0) {
22
- yield new KucoinFuturesSingleConnectionRealTimeFeed(exchange, wsFilters, this._httpURL, timeoutIntervalMS, onError)
28
+ yield new KucoinFuturesSingleConnectionRealTimeFeed(exchange, wsFilters, this._httpURL, timeoutIntervalMS, onError, validateSequence)
23
29
  }
24
30
 
25
31
  const contractDetailsFilters = filters.filter((f) => f.channel === 'contract/details')
@@ -35,9 +41,10 @@ export class KucoinFuturesSingleConnectionRealTimeFeed extends RealTimeFeedBase
35
41
  filters: Filter<string>[],
36
42
  private readonly _httpURL: string,
37
43
  timeoutIntervalMS: number | undefined,
38
- onError?: (error: Error) => void
44
+ onError?: (error: Error) => void,
45
+ validateSequence?: boolean
39
46
  ) {
40
- super(exchange, filters, timeoutIntervalMS, onError)
47
+ super(exchange, filters, timeoutIntervalMS, onError, validateSequence)
41
48
  }
42
49
  protected wssURL = ''
43
50
 
@@ -105,6 +112,18 @@ export class KucoinFuturesSingleConnectionRealTimeFeed extends RealTimeFeedBase
105
112
  return message.type === 'error'
106
113
  }
107
114
 
115
+ protected getSequence(message: any): Sequence | undefined {
116
+ if (message.subject === 'level2') {
117
+ const separatorIndex = message.topic.indexOf(':')
118
+ return {
119
+ key: message.topic.slice(separatorIndex + 1),
120
+ sequence: message.data.sequence,
121
+ previousSequence: message.data.sequence - 1
122
+ }
123
+ }
124
+ return undefined
125
+ }
126
+
108
127
  protected sendCustomPing = () => {
109
128
  this.send({
110
129
  id: new Date().valueOf().toString(),
@@ -1,5 +1,5 @@
1
1
  import { Filter } from '../types.ts'
2
- import { RealTimeFeedBase } from './realtimefeed.ts'
2
+ import { RealTimeFeedBase, type Sequence } from './realtimefeed.ts'
3
3
 
4
4
  abstract class RealTimeFeed extends RealTimeFeedBase {
5
5
  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
@@ -26,6 +26,17 @@ abstract class RealTimeFeed extends RealTimeFeedBase {
26
26
  protected messageIsError(message: any): boolean {
27
27
  return message.error !== undefined
28
28
  }
29
+
30
+ protected getSequence(message: any): Sequence | undefined {
31
+ if (message.type === 'subscribed/order_book' || message.type === 'update/order_book') {
32
+ return {
33
+ key: message.channel,
34
+ sequence: message.order_book.nonce,
35
+ previousSequence: message.type === 'update/order_book' ? message.order_book.begin_nonce : undefined
36
+ }
37
+ }
38
+ return undefined
39
+ }
29
40
  }
30
41
 
31
42
  export class LighterRealTimeFeed extends RealTimeFeed {
@@ -1,7 +1,7 @@
1
1
  import { CircularBuffer, getJSON, wait } from '../handy.ts'
2
2
  import type { MexcDepthSnapshotMessage } from '../mappers/mexc.ts'
3
3
  import { Filter } from '../types.ts'
4
- import { RealTimeFeedBase } from './realtimefeed.ts'
4
+ import { RealTimeFeedBase, type Sequence } from './realtimefeed.ts'
5
5
  import { decodeMexcProtobufMessage } from './mexcprotobuf.ts'
6
6
 
7
7
  export class MexcRealTimeFeed extends RealTimeFeedBase {
@@ -67,6 +67,17 @@ export class MexcRealTimeFeed extends RealTimeFeedBase {
67
67
  return message.msg === 'PONG'
68
68
  }
69
69
 
70
+ protected override getSequence(message: any): Sequence | undefined {
71
+ if (message.channel?.startsWith(`${MexcRealTimeFeed.depthChannel}@`) === true && message.publicAggreDepths !== undefined) {
72
+ return {
73
+ key: message.symbol,
74
+ sequence: Number(message.publicAggreDepths.toVersion),
75
+ previousSequence: Number(message.publicAggreDepths.fromVersion) - 1
76
+ }
77
+ }
78
+ return undefined
79
+ }
80
+
70
81
  protected override onMessage(message: any) {
71
82
  if (
72
83
  message.channel?.startsWith(MexcRealTimeFeed.depthChannel) !== true ||
@@ -1,7 +1,7 @@
1
1
  import { CircularBuffer, getJSON, wait } from '../handy.ts'
2
2
  import type { MexcFuturesDepthSnapshotData, MexcFuturesDepthSnapshotMessage } from '../mappers/mexcfutures.ts'
3
3
  import { Filter } from '../types.ts'
4
- import { RealTimeFeedBase } from './realtimefeed.ts'
4
+ import { RealTimeFeedBase, type Sequence } from './realtimefeed.ts'
5
5
 
6
6
  export class MexcFuturesRealTimeFeed extends RealTimeFeedBase {
7
7
  private readonly pendingDepthSnapshotSymbols = new Set<string>()
@@ -95,6 +95,18 @@ export class MexcFuturesRealTimeFeed extends RealTimeFeedBase {
95
95
  }
96
96
  }
97
97
 
98
+ protected override getSequence(message: any): Sequence | undefined {
99
+ const { data, channel, symbol } = message
100
+ if (channel === 'push.depth') {
101
+ return {
102
+ key: symbol,
103
+ sequence: data.end != null && data.end !== 0 ? data.end : data.version,
104
+ previousSequence: (data.begin != null && data.begin !== 0 ? data.begin : data.version) - 1
105
+ }
106
+ }
107
+ return undefined
108
+ }
109
+
98
110
  protected override onMessage(message: any) {
99
111
  if (
100
112
  message.channel !== 'push.depth' ||
@@ -2,20 +2,40 @@ import crypto from 'crypto'
2
2
  import { wait } from '../handy.ts'
3
3
  import { getOkexOptionsFamilyOrIndex, getOkexOptionsUnderlyingIndex } from '../okexsymbols.ts'
4
4
  import { Filter } from '../types.ts'
5
- import { MultiConnectionRealTimeFeedBase, RealTimeFeedBase } from './realtimefeed.ts'
5
+ import { MultiConnectionRealTimeFeedBase, RealTimeFeedBase, SequenceGapError, Sequence } from './realtimefeed.ts'
6
6
 
7
7
  export class OkexRealTimeFeed extends MultiConnectionRealTimeFeedBase {
8
- protected *_getRealTimeFeeds(exchange: string, filters: Filter<string>[], timeoutIntervalMS?: number, onError?: (error: Error) => void) {
8
+ protected *_getRealTimeFeeds(
9
+ exchange: string,
10
+ filters: Filter<string>[],
11
+ timeoutIntervalMS?: number,
12
+ onError?: (error: Error) => void,
13
+ validateSequence?: boolean
14
+ ) {
9
15
  const nonBusinessFilters = filters.filter((f) => f.channel !== 'trades-all')
10
16
 
11
17
  if (nonBusinessFilters.length > 0) {
12
- yield new OkexSingleRealTimeFeed('wss://ws.okx.com/ws/v5/public', exchange, nonBusinessFilters, timeoutIntervalMS, onError)
18
+ yield new OkexSingleRealTimeFeed(
19
+ 'wss://ws.okx.com/ws/v5/public',
20
+ exchange,
21
+ nonBusinessFilters,
22
+ timeoutIntervalMS,
23
+ onError,
24
+ validateSequence
25
+ )
13
26
  }
14
27
 
15
28
  const businessFilters = filters.filter((f) => f.channel === 'trades-all')
16
29
 
17
30
  if (businessFilters.length > 0) {
18
- yield new OkexSingleRealTimeFeed('wss://ws.okx.com/ws/v5/business', exchange, businessFilters, timeoutIntervalMS, onError)
31
+ yield new OkexSingleRealTimeFeed(
32
+ 'wss://ws.okx.com/ws/v5/business',
33
+ exchange,
34
+ businessFilters,
35
+ timeoutIntervalMS,
36
+ onError,
37
+ validateSequence
38
+ )
19
39
  }
20
40
  }
21
41
  }
@@ -26,13 +46,37 @@ class OkexSingleRealTimeFeed extends RealTimeFeedBase {
26
46
  exchange: string,
27
47
  filters: Filter<string>[],
28
48
  timeoutIntervalMS: number | undefined,
29
- onError?: (error: Error) => void
49
+ onError?: (error: Error) => void,
50
+ validateSequence?: boolean
30
51
  ) {
31
- super(exchange, filters, timeoutIntervalMS, onError)
52
+ super(exchange, filters, timeoutIntervalMS, onError, validateSequence)
32
53
  }
33
54
 
34
55
  private _hasCredentials = process.env.OKX_API_KEY !== undefined
35
56
 
57
+ protected getSequence(message: any): Sequence | undefined {
58
+ const { arg, action, data } = message
59
+ if (arg?.channel === 'books' && arg.instId && (action === 'snapshot' || action === 'update')) {
60
+ if (action === 'update') {
61
+ this.validateBookSequence(arg.instId, data)
62
+ }
63
+ return {
64
+ key: arg.instId,
65
+ sequence: data[data.length - 1].seqId,
66
+ previousSequence: action === 'snapshot' ? undefined : data[0].prevSeqId
67
+ }
68
+ }
69
+ return undefined
70
+ }
71
+
72
+ private validateBookSequence(symbol: string, data: any[]): void {
73
+ for (let i = 1; i < data.length; i++) {
74
+ if (data[i].prevSeqId !== data[i - 1].seqId) {
75
+ throw new SequenceGapError(this._exchange, symbol, data[i - 1].seqId, data[i].prevSeqId, data[i].seqId)
76
+ }
77
+ }
78
+ }
79
+
36
80
  private secondsSinceEpoch() {
37
81
  return Math.floor(Date.now() / 1000)
38
82
  }
@@ -103,17 +147,37 @@ class OkexSingleRealTimeFeed extends RealTimeFeedBase {
103
147
  }
104
148
 
105
149
  export class OkexOptionsRealTimeFeed extends MultiConnectionRealTimeFeedBase {
106
- protected *_getRealTimeFeeds(exchange: string, filters: Filter<string>[], timeoutIntervalMS?: number, onError?: (error: Error) => void) {
150
+ protected *_getRealTimeFeeds(
151
+ exchange: string,
152
+ filters: Filter<string>[],
153
+ timeoutIntervalMS?: number,
154
+ onError?: (error: Error) => void,
155
+ validateSequence?: boolean
156
+ ) {
107
157
  const nonBusinessFilters = filters.filter((f) => f.channel !== 'trades-all')
108
158
 
109
159
  if (nonBusinessFilters.length > 0) {
110
- yield new OkexOptionsSingleRealTimeFeed('wss://ws.okx.com/ws/v5/public', exchange, nonBusinessFilters, timeoutIntervalMS, onError)
160
+ yield new OkexOptionsSingleRealTimeFeed(
161
+ 'wss://ws.okx.com/ws/v5/public',
162
+ exchange,
163
+ nonBusinessFilters,
164
+ timeoutIntervalMS,
165
+ onError,
166
+ validateSequence
167
+ )
111
168
  }
112
169
 
113
170
  const businessFilters = filters.filter((f) => f.channel === 'trades-all')
114
171
 
115
172
  if (businessFilters.length > 0) {
116
- yield new OkexOptionsSingleRealTimeFeed('wss://ws.okx.com/ws/v5/business', exchange, businessFilters, timeoutIntervalMS, onError)
173
+ yield new OkexOptionsSingleRealTimeFeed(
174
+ 'wss://ws.okx.com/ws/v5/business',
175
+ exchange,
176
+ businessFilters,
177
+ timeoutIntervalMS,
178
+ onError,
179
+ validateSequence
180
+ )
117
181
  }
118
182
  }
119
183
  }
@@ -124,9 +188,10 @@ class OkexOptionsSingleRealTimeFeed extends OkexSingleRealTimeFeed {
124
188
  exchange: string,
125
189
  filters: Filter<string>[],
126
190
  timeoutIntervalMS: number | undefined,
127
- onError?: (error: Error) => void
191
+ onError?: (error: Error) => void,
192
+ validateSequence?: boolean
128
193
  ) {
129
- super(wssURL, exchange, filters, timeoutIntervalMS, onError)
194
+ super(wssURL, exchange, filters, timeoutIntervalMS, onError, validateSequence)
130
195
  }
131
196
 
132
197
  private _defaultIndexes = ['BTC-USD', 'ETH-USD', 'SOL-USD']
@@ -1,9 +1,21 @@
1
1
  import { Filter } from '../types.ts'
2
- import { RealTimeFeedBase } from './realtimefeed.ts'
2
+ import { RealTimeFeedBase, Sequence } from './realtimefeed.ts'
3
3
 
4
4
  export class PoloniexRealTimeFeed extends RealTimeFeedBase {
5
5
  protected readonly wssURL = 'wss://ws.poloniex.com/ws/public'
6
6
 
7
+ protected getSequence(message: any): Sequence | undefined {
8
+ const { channel, action, data } = message
9
+ if (channel === 'book_lv2' && (action === 'snapshot' || action === 'update')) {
10
+ return {
11
+ key: data[0].symbol,
12
+ sequence: data[0].id,
13
+ previousSequence: action === 'snapshot' ? undefined : data[0].lastId
14
+ }
15
+ }
16
+ return undefined
17
+ }
18
+
7
19
  protected mapToSubscribeMessages(filters: Filter<string>[]) {
8
20
  return filters.flatMap((filter) => {
9
21
  if (!filter.symbols || filter.symbols.length === 0) {
@@ -17,7 +17,8 @@ export class PolymarketRealTimeFeed extends MultiConnectionRealTimeFeedBase {
17
17
  exchange: string,
18
18
  filters: Filter<string>[],
19
19
  timeoutIntervalMS?: number,
20
- onError?: (error: Error) => void
20
+ onError?: (error: Error) => void,
21
+ validateSequence?: boolean
21
22
  ): IterableIterator<RealTimeFeedIterable> {
22
23
  const clobFilters: Filter<string>[] = []
23
24
  const sportsFilters: Filter<string>[] = []
@@ -37,10 +38,10 @@ export class PolymarketRealTimeFeed extends MultiConnectionRealTimeFeedBase {
37
38
  throw new Error('PolymarketRealTimeFeed requires explicitly specified symbols when subscribing to CLOB live feed')
38
39
  }
39
40
 
40
- yield new PolymarketClobRealTimeFeed(exchange, clobFilters, timeoutIntervalMS, onError)
41
+ yield new PolymarketClobRealTimeFeed(exchange, clobFilters, timeoutIntervalMS, onError, validateSequence)
41
42
  }
42
43
  if (sportsFilters.length > 0) {
43
- yield new PolymarketSportsRealTimeFeed(exchange, sportsFilters, timeoutIntervalMS, onError)
44
+ yield new PolymarketSportsRealTimeFeed(exchange, sportsFilters, timeoutIntervalMS, onError, validateSequence)
44
45
  }
45
46
  }
46
47
  }
@@ -13,7 +13,8 @@ export type RealTimeFeed = {
13
13
  exchange: Exchange,
14
14
  filters: Filter<string>[],
15
15
  timeoutIntervalMS: number | undefined,
16
- onError?: (error: Error) => void
16
+ onError?: (error: Error) => void,
17
+ validateSequence?: boolean
17
18
  ): RealTimeFeedIterable
18
19
  }
19
20
 
@@ -21,6 +22,26 @@ let connectionCounter = 1
21
22
 
22
23
  export type RealTimeFeedIterable = { [Symbol.asyncIterator](): ManagedRealTimeIterator<any> }
23
24
 
25
+ export type Sequence = {
26
+ key: string
27
+ sequence: number
28
+ previousSequence: number | undefined
29
+ }
30
+
31
+ export class SequenceGapError extends Error {
32
+ constructor(
33
+ readonly exchange: string,
34
+ readonly key: string,
35
+ readonly expectedPreviousSequence: number,
36
+ readonly previousSequence: number,
37
+ readonly sequence: number
38
+ ) {
39
+ var message = `${exchange} sequence gap on ${key}: expected previous sequence ${expectedPreviousSequence}, received ${previousSequence} (sequence ${sequence})`
40
+ super(message)
41
+ this.name = 'SequenceGapError'
42
+ }
43
+ }
44
+
24
45
  type RealTimeFeedConnection = {
25
46
  id: number
26
47
  ws: WebSocket
@@ -53,7 +74,8 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
53
74
  protected readonly _exchange: string,
54
75
  filters: Filter<string>[],
55
76
  private readonly _timeoutIntervalMS: number | undefined,
56
- private readonly _onError?: (error: Error) => void
77
+ private readonly _onError?: (error: Error) => void,
78
+ private readonly _validateSequence = true
57
79
  ) {
58
80
  this._filters = optimizeFilters(filters)
59
81
  this._parseJson = getMessageParser(_exchange)
@@ -90,6 +112,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
90
112
  let retries = 0
91
113
 
92
114
  while (this._closed === false) {
115
+ const sequences = new Map<string, number>()
93
116
  const connectionId = connectionCounter++
94
117
  let connection: RealTimeFeedConnection | undefined
95
118
  let staleConnectionTimerId: NodeJS.Timeout | undefined
@@ -172,6 +195,22 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
172
195
  currentConnection.receivedMessagesCount++
173
196
  }
174
197
 
198
+ if (this._validateSequence) {
199
+ const sequence = this.getSequence(messageDeserialized)
200
+ if (sequence !== undefined) {
201
+ const previous = sequences.get(sequence.key)
202
+ if (
203
+ previous !== undefined &&
204
+ sequence.previousSequence !== undefined &&
205
+ sequence.previousSequence !== previous &&
206
+ sequence.sequence !== previous
207
+ ) {
208
+ throw new SequenceGapError(this._exchange, sequence.key, previous, sequence.previousSequence, sequence.sequence)
209
+ }
210
+ sequences.set(sequence.key, sequence.sequence)
211
+ }
212
+ }
213
+
175
214
  this.onMessage(messageDeserialized)
176
215
 
177
216
  yield messageDeserialized
@@ -277,6 +316,10 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
277
316
 
278
317
  protected onMessage(_msg: any) {}
279
318
 
319
+ protected getSequence(_message: any): Sequence | undefined {
320
+ return undefined
321
+ }
322
+
280
323
  protected async onConnected() {}
281
324
 
282
325
  protected decompress?: (msg: any) => Buffer
@@ -464,7 +507,8 @@ export abstract class MultiConnectionRealTimeFeedBase implements RealTimeFeedIte
464
507
  private readonly _exchange: string,
465
508
  private readonly _filters: Filter<string>[],
466
509
  private readonly _timeoutIntervalMS: number | undefined,
467
- private readonly _onError?: (error: Error) => void
510
+ private readonly _onError?: (error: Error) => void,
511
+ private readonly _validateSequence = true
468
512
  ) {}
469
513
 
470
514
  [Symbol.asyncIterator](): ManagedRealTimeIterator<any> {
@@ -479,7 +523,10 @@ export abstract class MultiConnectionRealTimeFeedBase implements RealTimeFeedIte
479
523
 
480
524
  while (this._closeController.signal.aborted === false) {
481
525
  this._connections = mergeRealTime(
482
- Array.from(this._getRealTimeFeeds(this._exchange, this._filters, this._timeoutIntervalMS, this._onError), untilDisconnect)
526
+ Array.from(
527
+ this._getRealTimeFeeds(this._exchange, this._filters, this._timeoutIntervalMS, this._onError, this._validateSequence),
528
+ untilDisconnect
529
+ )
483
530
  )
484
531
  let disconnect: { error?: Error; delivered?: boolean } | undefined
485
532
 
@@ -512,7 +559,8 @@ export abstract class MultiConnectionRealTimeFeedBase implements RealTimeFeedIte
512
559
  exchange: string,
513
560
  filters: Filter<string>[],
514
561
  timeoutIntervalMS?: number,
515
- onError?: (error: Error) => void
562
+ onError?: (error: Error) => void,
563
+ validateSequence?: boolean
516
564
  ): IterableIterator<RealTimeFeedIterable>
517
565
  }
518
566
 
@@ -2,12 +2,18 @@ import { Writable } from 'stream'
2
2
  import { setTimeout as sleep } from 'node:timers/promises'
3
3
  import { batch, getJSON } from '../handy.ts'
4
4
  import { Filter } from '../types.ts'
5
- import { MultiConnectionRealTimeFeedBase, PoolingClientBase, RealTimeFeedBase } from './realtimefeed.ts'
5
+ import { MultiConnectionRealTimeFeedBase, PoolingClientBase, RealTimeFeedBase, Sequence } from './realtimefeed.ts'
6
6
 
7
7
  export class WhiteBitRealTimeFeed extends MultiConnectionRealTimeFeedBase {
8
8
  protected readonly httpURL: string = 'https://whitebit.com/api/v4/public'
9
9
 
10
- protected *_getRealTimeFeeds(exchange: string, filters: Filter<string>[], timeoutIntervalMS?: number, onError?: (error: Error) => void) {
10
+ protected *_getRealTimeFeeds(
11
+ exchange: string,
12
+ filters: Filter<string>[],
13
+ timeoutIntervalMS?: number,
14
+ onError?: (error: Error) => void,
15
+ validateSequence?: boolean
16
+ ) {
11
17
  filters = filters.flatMap((filter) => {
12
18
  const symbols = filter.symbols?.map((symbol) => symbol.toUpperCase())
13
19
  if (symbols?.length && (filter.channel === 'premiumIndex_update' || filter.channel === 'futures_update')) {
@@ -37,6 +43,7 @@ export class WhiteBitRealTimeFeed extends MultiConnectionRealTimeFeedBase {
37
43
  .filter((filter) => filter.symbols.length > 0),
38
44
  timeoutIntervalMS,
39
45
  onError,
46
+ validateSequence,
40
47
  connectionIndex++ * 50
41
48
  )
42
49
  }
@@ -57,16 +64,16 @@ export class WhiteBitRealTimeFeed extends MultiConnectionRealTimeFeedBase {
57
64
 
58
65
  export class WhiteBitWebSocketRealTimeFeed extends RealTimeFeedBase {
59
66
  protected readonly wssURL = 'wss://wss.whitebit.com/ws'
60
- private readonly _lastUpdateIds = new Map<string, number>()
61
67
 
62
68
  constructor(
63
69
  exchange: string,
64
70
  filters: Filter<string>[],
65
71
  timeoutIntervalMS: number | undefined,
66
72
  onError?: (error: Error) => void,
73
+ validateSequence?: boolean,
67
74
  private readonly _startDelayMS = 0
68
75
  ) {
69
- super(exchange, filters, timeoutIntervalMS, onError)
76
+ super(exchange, filters, timeoutIntervalMS, onError, validateSequence)
70
77
  }
71
78
 
72
79
  protected async getWebSocketUrl() {
@@ -75,7 +82,6 @@ export class WhiteBitWebSocketRealTimeFeed extends RealTimeFeedBase {
75
82
  }
76
83
 
77
84
  protected mapToSubscribeMessages(filters: Filter<string>[]) {
78
- this._lastUpdateIds.clear()
79
85
  let id = 0
80
86
  return filters.flatMap(({ channel, symbols }) => {
81
87
  const method = channel.replace('_update', '_subscribe')
@@ -110,16 +116,17 @@ export class WhiteBitWebSocketRealTimeFeed extends RealTimeFeedBase {
110
116
  return message.error !== undefined && message.error !== null
111
117
  }
112
118
 
113
- protected onMessage(message: { method?: string; params?: [boolean, { update_id: number; past_update_id?: number }, string] }) {
114
- if (message.method !== 'depth_update') {
115
- return
116
- }
117
- const [isSnapshot, book, symbol] = message.params!
118
- // Snapshot resets may jump over changes outside the subscribed depth; only deltas must link to the previous update.
119
- if (!isSnapshot && book.past_update_id !== this._lastUpdateIds.get(symbol)) {
120
- throw new Error(`WhiteBIT depth sequence gap for ${symbol}`)
119
+ protected getSequence(message: any): Sequence | undefined {
120
+ const { method, params } = message
121
+ if (method === 'depth_update') {
122
+ const [isSnapshot, book, symbol] = params
123
+ return {
124
+ key: symbol,
125
+ sequence: book.update_id,
126
+ previousSequence: isSnapshot ? undefined : book.past_update_id
127
+ }
121
128
  }
122
- this._lastUpdateIds.set(symbol, book.update_id)
129
+ return undefined
123
130
  }
124
131
  }
125
132
 
@@ -1,6 +1,6 @@
1
1
  import { getJSON } from '../handy.ts'
2
2
  import { Filter } from '../types.ts'
3
- import { RealTimeFeedBase } from './realtimefeed.ts'
3
+ import { RealTimeFeedBase, Sequence } from './realtimefeed.ts'
4
4
 
5
5
  const ORDER_BOOK_DEPTH = 50
6
6
 
@@ -8,6 +8,18 @@ export class WooxRealTimeFeed extends RealTimeFeedBase {
8
8
  protected wssURL = 'wss://wss.woox.io/v3/public'
9
9
  protected readonly httpURL = 'https://api.woox.io'
10
10
 
11
+ protected getSequence(message: any): Sequence | undefined {
12
+ const { topic, data } = message
13
+ if (topic?.startsWith('orderbookupdate@')) {
14
+ return {
15
+ key: data.s,
16
+ sequence: data.ts,
17
+ previousSequence: data.prevTs
18
+ }
19
+ }
20
+ return undefined
21
+ }
22
+
11
23
  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
12
24
  const symbolToTopics = new Map<string, string[]>()
13
25
 
package/src/stream.ts CHANGED
@@ -20,12 +20,19 @@ type NormalizedStreamState = {
20
20
  }
21
21
 
22
22
  async function* _stream<T extends Exchange, U extends boolean = false>(
23
- { exchange, filters, timeoutIntervalMS = 10000, withDisconnects = undefined, onError = undefined }: StreamOptions<T, U>,
23
+ {
24
+ exchange,
25
+ filters,
26
+ timeoutIntervalMS = 10000,
27
+ validateSequence = true,
28
+ withDisconnects = undefined,
29
+ onError = undefined
30
+ }: StreamOptions<T, U>,
24
31
  state: StreamState
25
32
  ): AsyncIterableIterator<U extends true ? { localTimestamp: Date; message: any } | undefined : { localTimestamp: Date; message: any }> {
26
33
  validateStreamOptions(filters)
27
34
 
28
- const realTimeFeed = createRealTimeFeed(exchange, filters, timeoutIntervalMS, onError)
35
+ const realTimeFeed = createRealTimeFeed(exchange, filters, timeoutIntervalMS, onError, validateSequence)
29
36
  const messages = realTimeFeed[Symbol.asyncIterator]()
30
37
  state.activeIterator = messages
31
38
 
@@ -62,13 +69,14 @@ export function stream<T extends Exchange, U extends boolean = false>({
62
69
  exchange,
63
70
  filters,
64
71
  timeoutIntervalMS = 10000,
72
+ validateSequence = true,
65
73
  withDisconnects = undefined,
66
74
  onError = undefined
67
75
  }: StreamOptions<T, U>): AsyncIterableIterator<
68
76
  U extends true ? { localTimestamp: Date; message: any } | undefined : { localTimestamp: Date; message: any }
69
77
  > {
70
78
  const state: StreamState = {}
71
- const iterator = _stream({ exchange, filters, timeoutIntervalMS, withDisconnects, onError }, state)
79
+ const iterator = _stream({ exchange, filters, timeoutIntervalMS, validateSequence, withDisconnects, onError }, state)
72
80
 
73
81
  return createManagedRealTimeIterator(iterator, () => {
74
82
  const activeIterator = state.activeIterator
@@ -78,7 +86,14 @@ export function stream<T extends Exchange, U extends boolean = false>({
78
86
  }
79
87
 
80
88
  async function* _streamNormalized<T extends Exchange, U extends MapperFactory<T, any>[], Z extends boolean = false>(
81
- { exchange, symbols, timeoutIntervalMS = 10000, withDisconnectMessages = undefined, onError = undefined }: StreamNormalizedOptions<T, Z>,
89
+ {
90
+ exchange,
91
+ symbols,
92
+ timeoutIntervalMS = 10000,
93
+ validateSequence = true,
94
+ withDisconnectMessages = undefined,
95
+ onError = undefined
96
+ }: StreamNormalizedOptions<T, Z>,
82
97
  state: NormalizedStreamState,
83
98
  ...normalizers: U
84
99
  ): AsyncIterableIterator<StreamNormalizedMessage<U, Z>> {
@@ -92,9 +107,10 @@ async function* _streamNormalized<T extends Exchange, U extends MapperFactory<T,
92
107
 
93
108
  const messages = stream({
94
109
  exchange,
95
- withDisconnects: true,
96
- timeoutIntervalMS,
97
110
  filters,
111
+ timeoutIntervalMS,
112
+ validateSequence,
113
+ withDisconnects: true,
98
114
  onError
99
115
  }) as ManagedRealTimeIterator<any>
100
116
  state.activeIterator = messages
@@ -160,6 +176,7 @@ export type StreamOptions<T extends Exchange, U extends boolean = false> = {
160
176
  exchange: T
161
177
  filters: FilterForExchange[T][]
162
178
  timeoutIntervalMS?: number
179
+ validateSequence?: boolean
163
180
  withDisconnects?: U
164
181
  onError?: (error: Error) => void
165
182
  }
@@ -168,16 +185,28 @@ export type StreamNormalizedOptions<T extends Exchange, U extends boolean = fals
168
185
  exchange: T
169
186
  symbols?: string[]
170
187
  timeoutIntervalMS?: number
188
+ validateSequence?: boolean
171
189
  withDisconnectMessages?: U
172
190
  onError?: (error: Error) => void
173
191
  }
174
192
 
175
193
  export function streamNormalized<T extends Exchange, U extends MapperFactory<T, any>[], Z extends boolean = false>(
176
- { exchange, symbols, timeoutIntervalMS = 10000, withDisconnectMessages = undefined, onError = undefined }: StreamNormalizedOptions<T, Z>,
194
+ {
195
+ exchange,
196
+ symbols,
197
+ timeoutIntervalMS = 10000,
198
+ validateSequence = true,
199
+ withDisconnectMessages = undefined,
200
+ onError = undefined
201
+ }: StreamNormalizedOptions<T, Z>,
177
202
  ...normalizers: U
178
203
  ): AsyncIterableIterator<StreamNormalizedMessage<U, Z>> {
179
204
  const state: NormalizedStreamState = { closed: false }
180
- const iterator = _streamNormalized({ exchange, symbols, timeoutIntervalMS, withDisconnectMessages, onError }, state, ...normalizers)
205
+ const iterator = _streamNormalized(
206
+ { exchange, symbols, timeoutIntervalMS, validateSequence, withDisconnectMessages, onError },
207
+ state,
208
+ ...normalizers
209
+ )
181
210
 
182
211
  return createManagedRealTimeIterator(iterator, () => {
183
212
  state.closed = true