tardis-dev 17.3.0 → 18.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boundedqueue.d.ts +19 -0
- package/dist/boundedqueue.d.ts.map +1 -0
- package/dist/boundedqueue.js +96 -0
- package/dist/boundedqueue.js.map +1 -0
- package/dist/combine.d.ts +6 -4
- package/dist/combine.d.ts.map +1 -1
- package/dist/combine.js +9 -34
- package/dist/combine.js.map +1 -1
- package/dist/computable/computable.d.ts.map +1 -1
- package/dist/computable/computable.js +6 -6
- package/dist/computable/computable.js.map +1 -1
- package/dist/downloaddatasets.d.ts.map +1 -1
- package/dist/downloaddatasets.js +4 -4
- package/dist/downloaddatasets.js.map +1 -1
- package/dist/foreachconcurrent.d.ts +2 -0
- package/dist/foreachconcurrent.d.ts.map +1 -0
- package/dist/foreachconcurrent.js +28 -0
- package/dist/foreachconcurrent.js.map +1 -0
- package/dist/realtimefeeds/binance.d.ts.map +1 -1
- package/dist/realtimefeeds/binance.js +9 -0
- package/dist/realtimefeeds/binance.js.map +1 -1
- package/dist/realtimefeeds/binancedex.d.ts.map +1 -1
- package/dist/realtimefeeds/binancedex.js +3 -0
- package/dist/realtimefeeds/binancedex.js.map +1 -1
- package/dist/realtimefeeds/bitstamp.d.ts.map +1 -1
- package/dist/realtimefeeds/bitstamp.js +3 -0
- package/dist/realtimefeeds/bitstamp.js.map +1 -1
- package/dist/realtimefeeds/huobi.d.ts.map +1 -1
- package/dist/realtimefeeds/huobi.js +3 -0
- package/dist/realtimefeeds/huobi.js.map +1 -1
- package/dist/realtimefeeds/index.d.ts +0 -1
- package/dist/realtimefeeds/index.d.ts.map +1 -1
- package/dist/realtimefeeds/index.js +0 -3
- package/dist/realtimefeeds/index.js.map +1 -1
- package/dist/realtimefeeds/kucoin.d.ts.map +1 -1
- package/dist/realtimefeeds/kucoin.js +3 -0
- package/dist/realtimefeeds/kucoin.js.map +1 -1
- package/dist/realtimefeeds/kucoinfutures.d.ts.map +1 -1
- package/dist/realtimefeeds/kucoinfutures.js +3 -0
- package/dist/realtimefeeds/kucoinfutures.js.map +1 -1
- package/dist/realtimefeeds/realtimefeed.d.ts +17 -8
- package/dist/realtimefeeds/realtimefeed.d.ts.map +1 -1
- package/dist/realtimefeeds/realtimefeed.js +186 -111
- package/dist/realtimefeeds/realtimefeed.js.map +1 -1
- package/dist/realtimefeeds/woox.d.ts.map +1 -1
- package/dist/realtimefeeds/woox.js +3 -0
- package/dist/realtimefeeds/woox.js.map +1 -1
- package/dist/realtimeiterator.d.ts +10 -0
- package/dist/realtimeiterator.d.ts.map +1 -0
- package/dist/realtimeiterator.js +106 -0
- package/dist/realtimeiterator.js.map +1 -0
- package/dist/stream.d.ts.map +1 -1
- package/dist/stream.js +50 -21
- package/dist/stream.js.map +1 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +3 -3
- package/dist/worker.js.map +1 -1
- package/package.json +1 -2
- package/src/boundedqueue.ts +118 -0
- package/src/combine.ts +16 -42
- package/src/computable/computable.ts +6 -5
- package/src/downloaddatasets.ts +4 -4
- package/src/foreachconcurrent.ts +29 -0
- package/src/realtimefeeds/binance.ts +10 -0
- package/src/realtimefeeds/binancedex.ts +3 -0
- package/src/realtimefeeds/bitstamp.ts +3 -0
- package/src/realtimefeeds/huobi.ts +4 -0
- package/src/realtimefeeds/index.ts +0 -4
- package/src/realtimefeeds/kucoin.ts +3 -0
- package/src/realtimefeeds/kucoinfutures.ts +3 -0
- package/src/realtimefeeds/realtimefeed.ts +230 -125
- package/src/realtimefeeds/woox.ts +3 -0
- package/src/realtimeiterator.ts +124 -0
- package/src/stream.ts +66 -33
- package/src/worker.ts +6 -10
|
@@ -355,6 +355,9 @@ class BinanceSingleConnectionRealTimeFeed extends RealTimeFeedBase {
|
|
|
355
355
|
}
|
|
356
356
|
|
|
357
357
|
const exchangeInfoResponse = await getJSON<any>(`${this._httpURL}/exchangeInfo`, binanceHttpOptions)
|
|
358
|
+
if (shouldCancel()) {
|
|
359
|
+
return
|
|
360
|
+
}
|
|
358
361
|
const exchangeInfo = exchangeInfoResponse.data
|
|
359
362
|
|
|
360
363
|
const DELAY_ENV = `${this._exchange.toUpperCase().replace(/-/g, '_')}_SNAPSHOTS_DELAY_MS`
|
|
@@ -408,10 +411,17 @@ class BinanceSingleConnectionRealTimeFeed extends RealTimeFeedBase {
|
|
|
408
411
|
await wait(delayMS)
|
|
409
412
|
}
|
|
410
413
|
|
|
414
|
+
if (shouldCancel()) {
|
|
415
|
+
return 0
|
|
416
|
+
}
|
|
417
|
+
|
|
411
418
|
const depthSnapshotResponse = await getJSON<any>(
|
|
412
419
|
`${this._httpURL}/depth?symbol=${symbol.toUpperCase()}&limit=1000`,
|
|
413
420
|
binanceHttpOptions
|
|
414
421
|
)
|
|
422
|
+
if (shouldCancel()) {
|
|
423
|
+
return 0
|
|
424
|
+
}
|
|
415
425
|
|
|
416
426
|
const snapshot = {
|
|
417
427
|
stream: `${symbol}@depthSnapshot`,
|
|
@@ -143,7 +143,3 @@ export function createRealTimeFeed(
|
|
|
143
143
|
|
|
144
144
|
return new RealTimeFeedFactory(exchange, filters, timeoutIntervalMS, onError)
|
|
145
145
|
}
|
|
146
|
-
|
|
147
|
-
export function setRealTimeFeedFactory(exchange: Exchange, realTimeFeed: RealTimeFeed) {
|
|
148
|
-
realTimeFeedsMap[exchange] = realTimeFeed
|
|
149
|
-
}
|
|
@@ -45,6 +45,9 @@ export class KucoinRealTimeFeed extends RealTimeFeedBase {
|
|
|
45
45
|
const { data } = await getJSON<any>(`${this._httpURL}/v1/market/orderbook/level2_100?symbol=${symbol}`, {
|
|
46
46
|
timeout: 10000
|
|
47
47
|
})
|
|
48
|
+
if (shouldCancel()) {
|
|
49
|
+
return
|
|
50
|
+
}
|
|
48
51
|
|
|
49
52
|
const snapshot = {
|
|
50
53
|
type: 'message',
|
|
@@ -77,6 +77,9 @@ export class KucoinFuturesSingleConnectionRealTimeFeed extends RealTimeFeedBase
|
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
const { data } = await getJSON<any>(`${this._httpURL}/v1/level2/snapshot?symbol=${symbol}`, kucoinHttpOptions)
|
|
80
|
+
if (shouldCancel()) {
|
|
81
|
+
return
|
|
82
|
+
}
|
|
80
83
|
|
|
81
84
|
const snapshot = {
|
|
82
85
|
type: 'message',
|
|
@@ -2,8 +2,9 @@ import dbg from 'debug'
|
|
|
2
2
|
import WebSocket, { createWebSocketStream } from 'ws'
|
|
3
3
|
import type { ClientRequestArgs } from 'http'
|
|
4
4
|
import { PassThrough, Writable } from 'stream'
|
|
5
|
-
import {
|
|
6
|
-
import { getProxyAgent, ONE_SEC_IN_MS, optimizeFilters
|
|
5
|
+
import { setTimeout as sleep } from 'node:timers/promises'
|
|
6
|
+
import { getProxyAgent, ONE_SEC_IN_MS, optimizeFilters } from '../handy.ts'
|
|
7
|
+
import { createManagedRealTimeIterator, mergeRealTime, type ManagedRealTimeIterator } from '../realtimeiterator.ts'
|
|
7
8
|
import { Exchange, Filter } from '../types.ts'
|
|
8
9
|
|
|
9
10
|
export type RealTimeFeed = {
|
|
@@ -19,9 +20,18 @@ let connectionCounter = 1
|
|
|
19
20
|
|
|
20
21
|
export type RealTimeFeedIterable = AsyncIterable<any>
|
|
21
22
|
|
|
23
|
+
type RealTimeFeedConnection = {
|
|
24
|
+
id: number
|
|
25
|
+
ws: WebSocket
|
|
26
|
+
controller: AbortController
|
|
27
|
+
receivedMessagesCount: number
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const ABORTED = Symbol('aborted')
|
|
31
|
+
|
|
22
32
|
export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
|
|
23
|
-
[Symbol.asyncIterator]() {
|
|
24
|
-
return this._stream()
|
|
33
|
+
[Symbol.asyncIterator](): ManagedRealTimeIterator<any> {
|
|
34
|
+
return createManagedRealTimeIterator(this._stream(), () => this._close())
|
|
25
35
|
}
|
|
26
36
|
|
|
27
37
|
protected readonly debug: dbg.Debugger
|
|
@@ -29,10 +39,10 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
|
|
|
29
39
|
protected readonly throttleSubscribeMS: number = 0
|
|
30
40
|
protected readonly manualSnapshotsBuffer: any[] = []
|
|
31
41
|
private readonly _filters: Filter<string>[]
|
|
32
|
-
private _receivedMessagesCount = 0
|
|
33
|
-
private _ws?: WebSocket
|
|
34
|
-
private _connectionId = -1
|
|
35
42
|
private _wsClientOptions: WebSocket.ClientOptions | ClientRequestArgs
|
|
43
|
+
private _closed = false
|
|
44
|
+
private readonly _closeController = new AbortController()
|
|
45
|
+
private _connection?: RealTimeFeedConnection
|
|
36
46
|
protected readonly originHeader: string | undefined = undefined
|
|
37
47
|
protected readonly extraHeaders: Record<string, string> | undefined = undefined
|
|
38
48
|
|
|
@@ -63,22 +73,40 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
|
|
|
63
73
|
return finalWssUrl
|
|
64
74
|
}
|
|
65
75
|
|
|
76
|
+
private _close() {
|
|
77
|
+
if (this._closed) {
|
|
78
|
+
return
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
this._closed = true
|
|
82
|
+
this._closeController.abort()
|
|
83
|
+
this._closeConnection(this._connection)
|
|
84
|
+
}
|
|
85
|
+
|
|
66
86
|
private async *_stream() {
|
|
67
|
-
let staleConnectionTimerId
|
|
68
|
-
let pingTimerId
|
|
69
87
|
let retries = 0
|
|
70
88
|
|
|
71
|
-
while (
|
|
89
|
+
while (this._closed === false) {
|
|
90
|
+
const connectionId = connectionCounter++
|
|
91
|
+
let connection: RealTimeFeedConnection | undefined
|
|
92
|
+
let staleConnectionTimerId: NodeJS.Timeout | undefined
|
|
93
|
+
let pingTimerId: NodeJS.Timeout | undefined
|
|
94
|
+
let connectionError: Error | undefined
|
|
95
|
+
|
|
72
96
|
try {
|
|
73
|
-
this.
|
|
97
|
+
this.manualSnapshotsBuffer.length = 0
|
|
74
98
|
const subscribeMessages = this.mapToSubscribeMessages(this._filters)
|
|
75
|
-
const finalWssUrl = await this.getWebSocketUrl()
|
|
99
|
+
const finalWssUrl = await resolveOrAbort(this.getWebSocketUrl(), this._closeController.signal)
|
|
100
|
+
|
|
101
|
+
if (finalWssUrl === ABORTED || this._closed) {
|
|
102
|
+
return
|
|
103
|
+
}
|
|
76
104
|
|
|
77
|
-
this.debug('(connection id: %d) estabilishing connection to %s',
|
|
105
|
+
this.debug('(connection id: %d) estabilishing connection to %s', connectionId, finalWssUrl)
|
|
78
106
|
|
|
79
107
|
this.debug(
|
|
80
108
|
'(connection id: %d) provided filters: %o mapped to subscribe messages: %j',
|
|
81
|
-
|
|
109
|
+
connectionId,
|
|
82
110
|
this._filters,
|
|
83
111
|
subscribeMessages
|
|
84
112
|
)
|
|
@@ -92,19 +120,31 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
|
|
|
92
120
|
}
|
|
93
121
|
|
|
94
122
|
this._wsClientOptions.agent = getProxyAgent(finalWssUrl)
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
123
|
+
const ws = new WebSocket(finalWssUrl, this._wsClientOptions)
|
|
124
|
+
const currentConnection: RealTimeFeedConnection = {
|
|
125
|
+
id: connectionId,
|
|
126
|
+
ws,
|
|
127
|
+
controller: new AbortController(),
|
|
128
|
+
receivedMessagesCount: 0
|
|
129
|
+
}
|
|
130
|
+
connection = currentConnection
|
|
131
|
+
this._connection = currentConnection
|
|
132
|
+
ws.onopen = () => this._onConnectionOpen(currentConnection, subscribeMessages)
|
|
133
|
+
ws.onclose = (event) => this._onConnectionClosed(currentConnection, event)
|
|
98
134
|
|
|
99
|
-
staleConnectionTimerId = this._monitorConnectionIfStale()
|
|
100
|
-
pingTimerId = this._sendPeriodicPing()
|
|
135
|
+
staleConnectionTimerId = this._monitorConnectionIfStale(currentConnection)
|
|
136
|
+
pingTimerId = this._sendPeriodicPing(currentConnection)
|
|
101
137
|
|
|
102
|
-
const realtimeMessagesStream = createWebSocketStream(
|
|
138
|
+
const realtimeMessagesStream = createWebSocketStream(ws, {
|
|
103
139
|
readableObjectMode: true, // othwerwise we may end up with multiple messages returned by stream in single iteration
|
|
104
140
|
readableHighWaterMark: 8096 // since we're in object mode, let's increase hwm a little from default of 16 messages buffered
|
|
105
141
|
}) as unknown as AsyncIterableIterator<Buffer>
|
|
106
142
|
|
|
107
143
|
for await (let message of realtimeMessagesStream) {
|
|
144
|
+
if (this._isConnectionActive(currentConnection) === false) {
|
|
145
|
+
return
|
|
146
|
+
}
|
|
147
|
+
|
|
108
148
|
if (this.decompress !== undefined) {
|
|
109
149
|
message = this.decompress(message)
|
|
110
150
|
}
|
|
@@ -129,7 +169,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
|
|
|
129
169
|
// exclude heaartbeat messages from received messages counter
|
|
130
170
|
// connection could still be stale even if only heartbeats are provided without any data
|
|
131
171
|
if (this.messageIsHeartbeat(messageDeserialized) === false) {
|
|
132
|
-
|
|
172
|
+
currentConnection.receivedMessagesCount++
|
|
133
173
|
}
|
|
134
174
|
|
|
135
175
|
this.onMessage(messageDeserialized)
|
|
@@ -150,80 +190,72 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
|
|
|
150
190
|
}
|
|
151
191
|
}
|
|
152
192
|
|
|
153
|
-
|
|
193
|
+
if (this._closed) {
|
|
194
|
+
return
|
|
195
|
+
}
|
|
196
|
+
} catch (error) {
|
|
197
|
+
if (this._closed) {
|
|
198
|
+
return
|
|
199
|
+
}
|
|
200
|
+
connectionError = error instanceof Error ? error : new Error(String(error))
|
|
201
|
+
} finally {
|
|
154
202
|
if (staleConnectionTimerId !== undefined) {
|
|
155
203
|
clearInterval(staleConnectionTimerId)
|
|
156
204
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
this._onError(error)
|
|
205
|
+
|
|
206
|
+
if (pingTimerId !== undefined) {
|
|
207
|
+
clearInterval(pingTimerId)
|
|
161
208
|
}
|
|
162
209
|
|
|
163
|
-
|
|
210
|
+
this._closeConnection(connection)
|
|
211
|
+
}
|
|
164
212
|
|
|
165
|
-
|
|
166
|
-
|
|
213
|
+
if (this._closed) {
|
|
214
|
+
return
|
|
215
|
+
}
|
|
167
216
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
delay = Math.pow(2, retries - 1) * 1000
|
|
217
|
+
let retryDelay: number | undefined
|
|
218
|
+
if (connectionError !== undefined) {
|
|
219
|
+
this._onError?.(connectionError)
|
|
220
|
+
retries++
|
|
173
221
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
}
|
|
222
|
+
const MAX_DELAY = 32 * 1000
|
|
223
|
+
const isRateLimited = connectionError.message.includes('429')
|
|
224
|
+
retryDelay = isRateLimited ? (MAX_DELAY / 2) * retries : Math.min(Math.pow(2, retries - 1) * 1000, MAX_DELAY)
|
|
178
225
|
|
|
179
226
|
this.debug(
|
|
180
227
|
'(connection id: %d) %s real-time feed connection error, retries count: %d, next retry delay: %dms, rate limited: %s error message: %o',
|
|
181
|
-
|
|
228
|
+
connectionId,
|
|
182
229
|
this._exchange,
|
|
183
230
|
retries,
|
|
184
|
-
|
|
231
|
+
retryDelay,
|
|
185
232
|
isRateLimited,
|
|
186
|
-
|
|
233
|
+
connectionError
|
|
187
234
|
)
|
|
235
|
+
}
|
|
188
236
|
|
|
189
|
-
|
|
190
|
-
if (staleConnectionTimerId !== undefined) {
|
|
191
|
-
clearInterval(staleConnectionTimerId)
|
|
192
|
-
}
|
|
193
|
-
yield { __disconnect__: true }
|
|
194
|
-
|
|
195
|
-
await wait(delay)
|
|
196
|
-
} finally {
|
|
197
|
-
// stop timers
|
|
198
|
-
if (staleConnectionTimerId !== undefined) {
|
|
199
|
-
clearInterval(staleConnectionTimerId)
|
|
200
|
-
}
|
|
237
|
+
yield { __disconnect__: true }
|
|
201
238
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
}
|
|
239
|
+
if (retryDelay !== undefined) {
|
|
240
|
+
await this._wait(retryDelay, this._closeController.signal)
|
|
205
241
|
}
|
|
206
242
|
}
|
|
207
243
|
}
|
|
208
244
|
|
|
209
245
|
protected send(msg: any) {
|
|
210
|
-
|
|
246
|
+
const ws = this._connection?.ws
|
|
247
|
+
if (ws === undefined || ws.readyState !== WebSocket.OPEN) {
|
|
211
248
|
return
|
|
212
249
|
}
|
|
213
|
-
|
|
214
|
-
return
|
|
215
|
-
}
|
|
216
|
-
this._ws.send(JSON.stringify(msg))
|
|
250
|
+
ws.send(JSON.stringify(msg))
|
|
217
251
|
}
|
|
218
252
|
|
|
219
253
|
protected sendRaw(msg: string | Buffer) {
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
}
|
|
223
|
-
if (this._ws.readyState !== WebSocket.OPEN) {
|
|
254
|
+
const ws = this._connection?.ws
|
|
255
|
+
if (ws === undefined || ws.readyState !== WebSocket.OPEN) {
|
|
224
256
|
return
|
|
225
257
|
}
|
|
226
|
-
|
|
258
|
+
ws.send(msg)
|
|
227
259
|
}
|
|
228
260
|
|
|
229
261
|
protected abstract mapToSubscribeMessages(filters: Filter<string>[]): any[]
|
|
@@ -252,48 +284,48 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
|
|
|
252
284
|
|
|
253
285
|
protected decompress?: (msg: any) => Buffer
|
|
254
286
|
|
|
255
|
-
private _monitorConnectionIfStale() {
|
|
287
|
+
private _monitorConnectionIfStale(connection: RealTimeFeedConnection) {
|
|
256
288
|
if (this._timeoutIntervalMS === undefined || this._timeoutIntervalMS === 0) {
|
|
257
289
|
return
|
|
258
290
|
}
|
|
259
291
|
|
|
260
292
|
// set up timer that checks against open, but stale connections that do not return any data
|
|
261
293
|
return setInterval(() => {
|
|
262
|
-
if (this.
|
|
294
|
+
if (this._isConnectionActive(connection) === false) {
|
|
263
295
|
return
|
|
264
296
|
}
|
|
265
297
|
|
|
266
|
-
if (
|
|
298
|
+
if (connection.receivedMessagesCount === 0) {
|
|
267
299
|
this.debug(
|
|
268
300
|
'(connection id: %d) did not received any messages within %d ms timeout, terminating connection...',
|
|
269
|
-
|
|
301
|
+
connection.id,
|
|
270
302
|
this._timeoutIntervalMS
|
|
271
303
|
)
|
|
272
|
-
|
|
304
|
+
connection.ws.terminate()
|
|
273
305
|
}
|
|
274
|
-
|
|
306
|
+
connection.receivedMessagesCount = 0
|
|
275
307
|
}, this._timeoutIntervalMS)
|
|
276
308
|
}
|
|
277
309
|
|
|
278
|
-
private _sendPeriodicPing() {
|
|
310
|
+
private _sendPeriodicPing(connection: RealTimeFeedConnection) {
|
|
279
311
|
return setInterval(() => {
|
|
280
|
-
if (this.
|
|
312
|
+
if (this._isConnectionOpen(connection) === false) {
|
|
281
313
|
return
|
|
282
314
|
}
|
|
283
315
|
|
|
284
316
|
if (this.sendCustomPing !== undefined) {
|
|
285
317
|
this.sendCustomPing()
|
|
286
318
|
} else {
|
|
287
|
-
|
|
319
|
+
connection.ws.ping()
|
|
288
320
|
}
|
|
289
321
|
}, 5 * ONE_SEC_IN_MS)
|
|
290
322
|
}
|
|
291
323
|
|
|
292
|
-
private
|
|
293
|
-
|
|
294
|
-
const subscribeMessages = this.mapToSubscribeMessages(this._filters)
|
|
324
|
+
private async _onConnectionEstablished(connection: RealTimeFeedConnection, subscribeMessages: any[]) {
|
|
325
|
+
const shouldCancel = () => this._isConnectionOpen(connection) === false
|
|
295
326
|
|
|
296
|
-
|
|
327
|
+
try {
|
|
328
|
+
const symbolsCount = this._filters.reduce((prev, curr) => {
|
|
297
329
|
if (curr.symbols !== undefined) {
|
|
298
330
|
for (const symbol of curr.symbols) {
|
|
299
331
|
prev.add(symbol)
|
|
@@ -304,77 +336,139 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
|
|
|
304
336
|
|
|
305
337
|
await this.onConnected()
|
|
306
338
|
|
|
339
|
+
if (shouldCancel()) {
|
|
340
|
+
return
|
|
341
|
+
}
|
|
342
|
+
|
|
307
343
|
for (const message of subscribeMessages) {
|
|
308
|
-
|
|
344
|
+
if (shouldCancel()) {
|
|
345
|
+
return
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
this._send(connection, message)
|
|
309
349
|
if (this.throttleSubscribeMS > 0) {
|
|
310
|
-
await
|
|
350
|
+
await this._wait(this.throttleSubscribeMS, connection.controller.signal)
|
|
311
351
|
}
|
|
312
352
|
}
|
|
313
353
|
|
|
314
|
-
this.debug('(connection id: %d) established connection',
|
|
354
|
+
this.debug('(connection id: %d) established connection', connection.id)
|
|
315
355
|
|
|
316
356
|
//wait before fetching snapshots until we're sure we've got proper connection estabilished (received some messages)
|
|
317
|
-
while (
|
|
318
|
-
await
|
|
357
|
+
while (shouldCancel() === false && connection.receivedMessagesCount < symbolsCount * 2) {
|
|
358
|
+
await this._wait(100, connection.controller.signal)
|
|
319
359
|
}
|
|
360
|
+
|
|
361
|
+
if (shouldCancel()) {
|
|
362
|
+
return
|
|
363
|
+
}
|
|
364
|
+
|
|
320
365
|
// wait a second just in case before starting fetching the snapshots
|
|
321
|
-
await
|
|
366
|
+
await this._wait(1 * ONE_SEC_IN_MS, connection.controller.signal)
|
|
322
367
|
|
|
323
|
-
if (
|
|
368
|
+
if (shouldCancel()) {
|
|
324
369
|
return
|
|
325
370
|
}
|
|
326
371
|
|
|
327
|
-
await this.provideManualSnapshots(this._filters,
|
|
372
|
+
await this.provideManualSnapshots(this._filters, shouldCancel)
|
|
328
373
|
} catch (e) {
|
|
329
|
-
this.debug('(connection id: %d) providing manual snapshots error: %o',
|
|
330
|
-
this.
|
|
374
|
+
this.debug('(connection id: %d) providing manual snapshots error: %o', connection.id, e)
|
|
375
|
+
if (this._isConnectionOpen(connection)) {
|
|
376
|
+
connection.ws.emit('error', e)
|
|
377
|
+
}
|
|
331
378
|
}
|
|
332
379
|
}
|
|
333
380
|
|
|
334
|
-
private _onConnectionClosed
|
|
335
|
-
this.debug('(connection id: %d) connection closed %s',
|
|
381
|
+
private _onConnectionClosed(connection: RealTimeFeedConnection, event: WebSocket.CloseEvent) {
|
|
382
|
+
this.debug('(connection id: %d) connection closed %s', connection.id, event.reason)
|
|
336
383
|
}
|
|
337
|
-
}
|
|
338
384
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
private readonly _timeoutIntervalMS: number | undefined,
|
|
344
|
-
private readonly _onError?: (error: Error) => void
|
|
345
|
-
) {}
|
|
385
|
+
private _onConnectionOpen(connection: RealTimeFeedConnection, subscribeMessages: any[]) {
|
|
386
|
+
if (this._isConnectionActive(connection) === false) {
|
|
387
|
+
return
|
|
388
|
+
}
|
|
346
389
|
|
|
347
|
-
|
|
348
|
-
return this._stream()
|
|
390
|
+
void this._onConnectionEstablished(connection, subscribeMessages)
|
|
349
391
|
}
|
|
350
392
|
|
|
351
|
-
private
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
393
|
+
private _send(connection: RealTimeFeedConnection, message: any) {
|
|
394
|
+
if (this._isConnectionOpen(connection)) {
|
|
395
|
+
connection.ws.send(JSON.stringify(message))
|
|
396
|
+
}
|
|
397
|
+
}
|
|
356
398
|
|
|
357
|
-
|
|
399
|
+
private _isConnectionActive(connection: RealTimeFeedConnection) {
|
|
400
|
+
return this._closed === false && this._connection === connection && connection.controller.signal.aborted === false
|
|
401
|
+
}
|
|
358
402
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
for await (const message of realTimeFeed) {
|
|
363
|
-
if (combinedStream.destroyed) {
|
|
364
|
-
return
|
|
365
|
-
}
|
|
403
|
+
private _isConnectionOpen(connection: RealTimeFeedConnection) {
|
|
404
|
+
return this._isConnectionActive(connection) && connection.ws.readyState === WebSocket.OPEN
|
|
405
|
+
}
|
|
366
406
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
407
|
+
private _closeConnection(connection: RealTimeFeedConnection | undefined) {
|
|
408
|
+
if (connection === undefined) {
|
|
409
|
+
return
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
connection.controller.abort()
|
|
413
|
+
if (connection.ws.readyState !== WebSocket.CLOSED) {
|
|
414
|
+
connection.ws.terminate()
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
if (this._connection === connection) {
|
|
418
|
+
this._connection = undefined
|
|
419
|
+
this.manualSnapshotsBuffer.length = 0
|
|
372
420
|
}
|
|
421
|
+
}
|
|
373
422
|
|
|
374
|
-
|
|
375
|
-
|
|
423
|
+
private async _wait(delayMS: number, signal: AbortSignal) {
|
|
424
|
+
try {
|
|
425
|
+
await sleep(delayMS, undefined, { signal })
|
|
426
|
+
} catch (error) {
|
|
427
|
+
if (signal.aborted === false) {
|
|
428
|
+
throw error
|
|
429
|
+
}
|
|
376
430
|
}
|
|
377
431
|
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
function resolveOrAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T | typeof ABORTED> {
|
|
435
|
+
if (signal.aborted) {
|
|
436
|
+
return Promise.resolve(ABORTED)
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
return new Promise((resolve, reject) => {
|
|
440
|
+
let settled = false
|
|
441
|
+
const onAbort = () => settle(() => resolve(ABORTED))
|
|
442
|
+
const settle = (callback: () => void) => {
|
|
443
|
+
if (settled) {
|
|
444
|
+
return
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
settled = true
|
|
448
|
+
signal.removeEventListener('abort', onAbort)
|
|
449
|
+
callback()
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
signal.addEventListener('abort', onAbort, { once: true })
|
|
453
|
+
void promise.then(
|
|
454
|
+
(value) => settle(() => resolve(value)),
|
|
455
|
+
(error) => settle(() => reject(error))
|
|
456
|
+
)
|
|
457
|
+
})
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
export abstract class MultiConnectionRealTimeFeedBase implements RealTimeFeedIterable {
|
|
461
|
+
constructor(
|
|
462
|
+
private readonly _exchange: string,
|
|
463
|
+
private readonly _filters: Filter<string>[],
|
|
464
|
+
private readonly _timeoutIntervalMS: number | undefined,
|
|
465
|
+
private readonly _onError?: (error: Error) => void
|
|
466
|
+
) {}
|
|
467
|
+
|
|
468
|
+
[Symbol.asyncIterator](): ManagedRealTimeIterator<any> {
|
|
469
|
+
const realTimeFeeds = Array.from(this._getRealTimeFeeds(this._exchange, this._filters, this._timeoutIntervalMS, this._onError))
|
|
470
|
+
return mergeRealTime(realTimeFeeds)
|
|
471
|
+
}
|
|
378
472
|
|
|
379
473
|
protected abstract _getRealTimeFeeds(
|
|
380
474
|
exchange: string,
|
|
@@ -387,6 +481,7 @@ export abstract class MultiConnectionRealTimeFeedBase implements RealTimeFeedIte
|
|
|
387
481
|
export abstract class PoolingClientBase implements RealTimeFeedIterable {
|
|
388
482
|
protected readonly debug: dbg.Debugger
|
|
389
483
|
private _tid: NodeJS.Timeout | undefined = undefined
|
|
484
|
+
private _outputStream: PassThrough | undefined
|
|
390
485
|
|
|
391
486
|
constructor(
|
|
392
487
|
exchange: string,
|
|
@@ -396,8 +491,8 @@ export abstract class PoolingClientBase implements RealTimeFeedIterable {
|
|
|
396
491
|
this.debug = dbg(`tardis-dev:pooling-client:${exchange}`)
|
|
397
492
|
}
|
|
398
493
|
|
|
399
|
-
[Symbol.asyncIterator]() {
|
|
400
|
-
return this._stream()
|
|
494
|
+
[Symbol.asyncIterator](): ManagedRealTimeIterator<any> {
|
|
495
|
+
return createManagedRealTimeIterator(this._stream(), () => this._close())
|
|
401
496
|
}
|
|
402
497
|
|
|
403
498
|
protected abstract poolDataToStream(outputStream: Writable): Promise<void>
|
|
@@ -406,6 +501,14 @@ export abstract class PoolingClientBase implements RealTimeFeedIterable {
|
|
|
406
501
|
return this._poolingIntervalSeconds * ONE_SEC_IN_MS
|
|
407
502
|
}
|
|
408
503
|
|
|
504
|
+
private _close() {
|
|
505
|
+
if (this._tid !== undefined) {
|
|
506
|
+
clearTimeout(this._tid)
|
|
507
|
+
this._tid = undefined
|
|
508
|
+
}
|
|
509
|
+
this._outputStream?.destroy()
|
|
510
|
+
}
|
|
511
|
+
|
|
409
512
|
private async _startPooling(outputStream: Writable) {
|
|
410
513
|
const pool = async () => {
|
|
411
514
|
try {
|
|
@@ -436,6 +539,7 @@ export abstract class PoolingClientBase implements RealTimeFeedIterable {
|
|
|
436
539
|
objectMode: true,
|
|
437
540
|
highWaterMark: 1024
|
|
438
541
|
})
|
|
542
|
+
this._outputStream = stream
|
|
439
543
|
|
|
440
544
|
this._startPooling(stream)
|
|
441
545
|
|
|
@@ -446,8 +550,9 @@ export abstract class PoolingClientBase implements RealTimeFeedIterable {
|
|
|
446
550
|
yield message
|
|
447
551
|
}
|
|
448
552
|
} finally {
|
|
449
|
-
|
|
450
|
-
|
|
553
|
+
this._close()
|
|
554
|
+
if (this._outputStream === stream) {
|
|
555
|
+
this._outputStream = undefined
|
|
451
556
|
}
|
|
452
557
|
|
|
453
558
|
this.debug('pooling finished')
|
|
@@ -46,6 +46,9 @@ export class WooxRealTimeFeed extends RealTimeFeedBase {
|
|
|
46
46
|
const response = await getJSON<WooxOrderBookSnapshotResponse>(
|
|
47
47
|
`${this.httpURL}/v3/public/orderbook?symbol=${symbol}&maxLevel=${ORDER_BOOK_DEPTH}`
|
|
48
48
|
)
|
|
49
|
+
if (shouldCancel()) {
|
|
50
|
+
return
|
|
51
|
+
}
|
|
49
52
|
|
|
50
53
|
if (response.data.success === false) {
|
|
51
54
|
throw new Error(`Woox orderbook snapshot request failed: ${JSON.stringify(response.data)}`)
|