tardis-dev 12.5.22 → 12.6.7

package/src/realtimefeeds/dydx.ts

@@ -1,40 +1,40 @@
-import { Filter } from '../types'
-import { RealTimeFeedBase } from './realtimefeed'
-
-export class DydxRealTimeFeed extends RealTimeFeedBase {
-  protected readonly wssURL = 'wss://api.dydx.exchange/v3/ws'
-
-  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
-    const subs = filters
-      .map((filter) => {
-        if (filter.channel === 'v3_markets') {
-          return [
-            {
-              type: 'subscribe',
-              channel: 'v3_markets'
-            }
-          ]
-        }
-
-        if (!filter.symbols || filter.symbols.length === 0) {
-          throw new Error('DydxRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
-        }
-
-        return filter.symbols.map((symbol) => {
-          return {
-            type: 'subscribe',
-            channel: filter.channel,
-            id: symbol,
-            includeOffsets: filter.channel === 'v3_orderbook' ? true : undefined
-          }
-        })
-      })
-      .flatMap((f) => f)
-
-    return subs
-  }
-
-  protected messageIsError(message: any): boolean {
-    return message.type === 'error'
-  }
-}
+import { Filter } from '../types'
+import { RealTimeFeedBase } from './realtimefeed'
+
+export class DydxRealTimeFeed extends RealTimeFeedBase {
+  protected readonly wssURL = 'wss://api.dydx.exchange/v3/ws'
+
+  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
+    const subs = filters
+      .map((filter) => {
+        if (filter.channel === 'v3_markets') {
+          return [
+            {
+              type: 'subscribe',
+              channel: 'v3_markets'
+            }
+          ]
+        }
+
+        if (!filter.symbols || filter.symbols.length === 0) {
+          throw new Error('DydxRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
+        }
+
+        return filter.symbols.map((symbol) => {
+          return {
+            type: 'subscribe',
+            channel: filter.channel,
+            id: symbol,
+            includeOffsets: filter.channel === 'v3_orderbook' ? true : undefined
+          }
+        })
+      })
+      .flatMap((f) => f)
+
+    return subs
+  }
+
+  protected messageIsError(message: any): boolean {
+    return message.type === 'error'
+  }
+}

package/src/realtimefeeds/gateio.ts

@@ -1,41 +1,41 @@
-import { Filter } from '../types'
-import { RealTimeFeedBase } from './realtimefeed'
-
-export class GateIORealTimeFeed extends RealTimeFeedBase {
-  protected readonly wssURL = 'wss://ws.gate.io/v3/'
-
-  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
-    const id = 1
-    const payload = filters.map((filter) => {
-      if (!filter.symbols || filter.symbols.length === 0) {
-        throw new Error('GateIORealTimeFeed requires explicitly specified symbols when subscribing to live feed')
-      }
-
-      if (filter.channel === 'depth') {
-        return {
-          id,
-          method: `${filter.channel}.subscribe`,
-          params: filter.symbols.map((s) => {
-            return [s, 30, '0']
-          })
-        }
-      } else {
-        return {
-          id,
-          method: `${filter.channel}.subscribe`,
-          params: filter.symbols
-        }
-      }
-    })
-
-    return payload
-  }
-
-  protected messageIsError(message: any): boolean {
-    if (message.error !== null && message.error !== undefined) {
-      return true
-    }
-
-    return false
-  }
-}
+import { Filter } from '../types'
+import { RealTimeFeedBase } from './realtimefeed'
+
+export class GateIORealTimeFeed extends RealTimeFeedBase {
+  protected readonly wssURL = 'wss://ws.gate.io/v3/'
+
+  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
+    const id = 1
+    const payload = filters.map((filter) => {
+      if (!filter.symbols || filter.symbols.length === 0) {
+        throw new Error('GateIORealTimeFeed requires explicitly specified symbols when subscribing to live feed')
+      }
+
+      if (filter.channel === 'depth') {
+        return {
+          id,
+          method: `${filter.channel}.subscribe`,
+          params: filter.symbols.map((s) => {
+            return [s, 30, '0']
+          })
+        }
+      } else {
+        return {
+          id,
+          method: `${filter.channel}.subscribe`,
+          params: filter.symbols
+        }
+      }
+    })
+
+    return payload
+  }
+
+  protected messageIsError(message: any): boolean {
+    if (message.error !== null && message.error !== undefined) {
+      return true
+    }
+
+    return false
+  }
+}

package/src/realtimefeeds/gateiofutures.ts

@@ -1,90 +1,90 @@
-import { Filter } from '../types'
-import { RealTimeFeedBase, MultiConnectionRealTimeFeedBase } from './realtimefeed'
-
-export class GateIOFuturesRealTimeFeed extends MultiConnectionRealTimeFeedBase {
-  protected *_getRealTimeFeeds(exchange: string, filters: Filter<string>[], timeoutIntervalMS?: number, onError?: (error: Error) => void) {
-    const linearContractsFilters = filters.reduce(
-      this._only((s) => s.endsWith('_USDT')),
-      [] as Filter<string>[]
-    )
-
-    const inverseContractsFilters = filters.reduce(
-      this._only((s) => s.endsWith('_USDT') === false),
-      [] as Filter<string>[]
-    )
-
-    if (linearContractsFilters.length > 0) {
-      yield new GateIOFuturesSingleConnectionRealTimeFeed('usdt', exchange, linearContractsFilters, timeoutIntervalMS, onError)
-    }
-
-    if (inverseContractsFilters.length > 0) {
-      yield new GateIOFuturesSingleConnectionRealTimeFeed('btc', exchange, inverseContractsFilters, timeoutIntervalMS, onError)
-    }
-  }
-
-  private _only(filter: (symbol: string) => boolean) {
-    return (prev: Filter<string>[], current: Filter<string>) => {
-      if (!current.symbols || current.symbols.length === 0) {
-        throw new Error('GateIOFuturesRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
-      }
-
-      const symbols = current.symbols.filter(filter)
-      if (symbols.length > 0) {
-        prev.push({
-          channel: current.channel,
-          symbols
-        })
-      }
-      return prev
-    }
-  }
-}
-
-class GateIOFuturesSingleConnectionRealTimeFeed extends RealTimeFeedBase {
-  protected readonly wssURL: string
-
-  constructor(
-    wsURLSuffix: string,
-    exchange: string,
-    filters: Filter<string>[],
-    timeoutIntervalMS?: number,
-    onError?: (error: Error) => void
-  ) {
-    super(exchange, filters, timeoutIntervalMS, onError)
-    this.wssURL = `wss://fx-ws.gateio.ws/v4/ws/${wsURLSuffix}`
-  }
-
-  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
-    const payload = filters.flatMap((filter) => {
-      if (filter.channel === 'order_book') {
-        return filter.symbols!.map((symbol) => {
-          return {
-            event: 'subscribe',
-            channel: `futures.${filter.channel}`,
-            payload: [symbol, '20', '0'],
-            time: Math.floor(new Date().valueOf() / 1000)
-          }
-        })
-      } else {
-        return [
-          {
-            event: 'subscribe',
-            channel: `futures.${filter.channel}`,
-            payload: filter.symbols,
-            time: Math.floor(new Date().valueOf() / 1000)
-          }
-        ]
-      }
-    })
-
-    return payload
-  }
-
-  protected messageIsError(message: any): boolean {
-    if (message.error !== null && message.error !== undefined) {
-      return true
-    }
-
-    return false
-  }
-}
+import { Filter } from '../types'
+import { RealTimeFeedBase, MultiConnectionRealTimeFeedBase } from './realtimefeed'
+
+export class GateIOFuturesRealTimeFeed extends MultiConnectionRealTimeFeedBase {
+  protected *_getRealTimeFeeds(exchange: string, filters: Filter<string>[], timeoutIntervalMS?: number, onError?: (error: Error) => void) {
+    const linearContractsFilters = filters.reduce(
+      this._only((s) => s.endsWith('_USDT')),
+      [] as Filter<string>[]
+    )
+
+    const inverseContractsFilters = filters.reduce(
+      this._only((s) => s.endsWith('_USDT') === false),
+      [] as Filter<string>[]
+    )
+
+    if (linearContractsFilters.length > 0) {
+      yield new GateIOFuturesSingleConnectionRealTimeFeed('usdt', exchange, linearContractsFilters, timeoutIntervalMS, onError)
+    }
+
+    if (inverseContractsFilters.length > 0) {
+      yield new GateIOFuturesSingleConnectionRealTimeFeed('btc', exchange, inverseContractsFilters, timeoutIntervalMS, onError)
+    }
+  }
+
+  private _only(filter: (symbol: string) => boolean) {
+    return (prev: Filter<string>[], current: Filter<string>) => {
+      if (!current.symbols || current.symbols.length === 0) {
+        throw new Error('GateIOFuturesRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
+      }
+
+      const symbols = current.symbols.filter(filter)
+      if (symbols.length > 0) {
+        prev.push({
+          channel: current.channel,
+          symbols
+        })
+      }
+      return prev
+    }
+  }
+}
+
+class GateIOFuturesSingleConnectionRealTimeFeed extends RealTimeFeedBase {
+  protected readonly wssURL: string
+
+  constructor(
+    wsURLSuffix: string,
+    exchange: string,
+    filters: Filter<string>[],
+    timeoutIntervalMS?: number,
+    onError?: (error: Error) => void
+  ) {
+    super(exchange, filters, timeoutIntervalMS, onError)
+    this.wssURL = `wss://fx-ws.gateio.ws/v4/ws/${wsURLSuffix}`
+  }
+
+  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
+    const payload = filters.flatMap((filter) => {
+      if (filter.channel === 'order_book') {
+        return filter.symbols!.map((symbol) => {
+          return {
+            event: 'subscribe',
+            channel: `futures.${filter.channel}`,
+            payload: [symbol, '20', '0'],
+            time: Math.floor(new Date().valueOf() / 1000)
+          }
+        })
+      } else {
+        return [
+          {
+            event: 'subscribe',
+            channel: `futures.${filter.channel}`,
+            payload: filter.symbols,
+            time: Math.floor(new Date().valueOf() / 1000)
+          }
+        ]
+      }
+    })
+
+    return payload
+  }
+
+  protected messageIsError(message: any): boolean {
+    if (message.error !== null && message.error !== undefined) {
+      return true
+    }
+
+    return false
+  }
+}

package/src/realtimefeeds/poloniex.ts

@@ -1,28 +1,28 @@
-import { Filter } from '../types'
-import { RealTimeFeedBase } from './realtimefeed'
-
-export class PoloniexRealTimeFeed extends RealTimeFeedBase {
-  protected readonly wssURL = 'wss://api2.poloniex.com'
-
-  protected mapToSubscribeMessages(filters: Filter<string>[]) {
-    const allSymbols = filters.flatMap((filter) => {
-      if (!filter.symbols || filter.symbols.length === 0) {
-        throw new Error('PoloniexRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
-      }
-      return filter.symbols
-    })
-
-    const uniqueSymbols = [...new Set(allSymbols)]
-
-    return uniqueSymbols.map((symbol) => {
-      return {
-        command: 'subscribe',
-        channel: symbol
-      }
-    })
-  }
-
-  protected messageIsError(message: any): boolean {
-    return message.error !== undefined && message.error !== null
-  }
-}
+import { Filter } from '../types'
+import { RealTimeFeedBase } from './realtimefeed'
+
+export class PoloniexRealTimeFeed extends RealTimeFeedBase {
+  protected readonly wssURL = 'wss://api2.poloniex.com'
+
+  protected mapToSubscribeMessages(filters: Filter<string>[]) {
+    const allSymbols = filters.flatMap((filter) => {
+      if (!filter.symbols || filter.symbols.length === 0) {
+        throw new Error('PoloniexRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
+      }
+      return filter.symbols
+    })
+
+    const uniqueSymbols = [...new Set(allSymbols)]
+
+    return uniqueSymbols.map((symbol) => {
+      return {
+        command: 'subscribe',
+        channel: symbol
+      }
+    })
+  }
+
+  protected messageIsError(message: any): boolean {
+    return message.error !== undefined && message.error !== null
+  }
+}

package/src/realtimefeeds/realtimefeed.ts

@@ -43,7 +43,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
     this._filters = optimizeFilters(filters)
     this.debug = dbg(`tardis-dev:realtime:${_exchange}`)
 
-    this._wsClientOptions = { perMessageDeflate: false, handshakeTimeout: 10 * ONE_SEC_IN_MS }
+    this._wsClientOptions = { perMessageDeflate: false, handshakeTimeout: 10 * ONE_SEC_IN_MS, skipUTF8Validation: true } as any
 
     if (httpsProxyAgent !== undefined) {
       this._wsClientOptions.agent = httpsProxyAgent
@@ -123,7 +123,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
           clearInterval(staleConnectionTimerId)
         }
         yield { __disconnect__: true }
-      } catch (error) {
+      } catch (error: any) {
         if (this._onError !== undefined) {
           this._onError(error)
         }

package/src/realtimefeeds/upbit.ts

@@ -1,35 +1,35 @@
-import { Filter } from '../types'
-import { RealTimeFeedBase } from './realtimefeed'
-
-export class UpbitRealTimeFeed extends RealTimeFeedBase {
-  protected readonly wssURL = 'wss://api.upbit.com/websocket/v1'
-
-  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
-    const subs = filters.map((filter) => {
-      if (!filter.symbols || filter.symbols.length === 0) {
-        throw new Error('UpbitRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
-      }
-
-      return {
-        type: filter.channel,
-        codes: filter.symbols,
-        isOnlyRealtime: true
-      }
-    })
-
-    const payload = [
-      [
-        {
-          ticket: new Date().valueOf().toString()
-        },
-        ...subs
-      ]
-    ]
-
-    return payload
-  }
-
-  protected messageIsError(message: any): boolean {
-    return message.type === 'error'
-  }
-}
+import { Filter } from '../types'
+import { RealTimeFeedBase } from './realtimefeed'
+
+export class UpbitRealTimeFeed extends RealTimeFeedBase {
+  protected readonly wssURL = 'wss://api.upbit.com/websocket/v1'
+
+  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
+    const subs = filters.map((filter) => {
+      if (!filter.symbols || filter.symbols.length === 0) {
+        throw new Error('UpbitRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
+      }
+
+      return {
+        type: filter.channel,
+        codes: filter.symbols,
+        isOnlyRealtime: true
+      }
+    })
+
+    const payload = [
+      [
+        {
+          ticket: new Date().valueOf().toString()
+        },
+        ...subs
+      ]
+    ]
+
+    return payload
+  }
+
+  protected messageIsError(message: any): boolean {
+    return message.type === 'error'
+  }
+}

package/src/replay.ts

@@ -1,16 +1,17 @@
 import { createReadStream } from 'fs-extra'
 import path from 'path'
+import { EventEmitter } from 'stream'
 import { Worker } from 'worker_threads'
 import { constants, createGunzip } from 'zlib'
 import { BinarySplitStream } from './binarysplit'
+import { clearCacheSync } from './clearcache'
 import { EXCHANGES, EXCHANGE_CHANNELS_INFO } from './consts'
 import { debug } from './debug'
-import { getFilters, normalizeMessages, parseAsUTCDate, parseμs, wait, addDays } from './handy'
+import { addDays, getFilters, normalizeMessages, parseAsUTCDate, parseμs, wait } from './handy'
 import { MapperFactory, normalizeBookChanges } from './mappers'
 import { getOptions } from './options'
 import { Disconnect, Exchange, FilterForExchange } from './types'
 import { WorkerJobPayload, WorkerMessage, WorkerSignal } from './worker'
-import { clearCacheSync } from './clearcache'
 
 export async function* replay<T extends Exchange, U extends boolean = false, Z extends boolean = false>({
   exchange,
@@ -55,9 +56,7 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
     waitWhenDataNotYetAvailable
   }
 
-  const worker = new Worker(path.resolve(__dirname, 'worker.js'), {
-    workerData: payload
-  })
+  const worker = new ReliableWorker(payload)
 
   worker.on('message', (message: WorkerMessage) => {
     cachedSlicePaths.set(message.sliceKey, message.slicePath)
@@ -69,25 +68,20 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
     replayError = err
   })
 
-  worker.on('exit', (code) => {
-    debug('worker finished with code: %d', code)
-  })
-
   try {
     // date is always formatted to have length of 28 so we can skip looking for first space in line and use it
     // as hardcoded value
     const DATE_MESSAGE_SPLIT_INDEX = 28
 
-    // experimental more lenient gzip decompression, behind env flag for now
+    // more lenient gzip decompression
     // see https://github.com/request/request/pull/2492 and https://github.com/node-fetch/node-fetch/pull/239
 
-    const ZLIB_OPTIONS = process.env.TARDIS_LENIENT_GZIP_DECOMPRESS
-      ? {
-          chunkSize: 128 * 1024,
-          flush: constants.Z_SYNC_FLUSH,
-          finishFlush: constants.Z_SYNC_FLUSH
-        }
-      : { chunkSize: 128 * 1024 }
+    const ZLIB_OPTIONS = {
+      chunkSize: 128 * 1024,
+      flush: constants.Z_SYNC_FLUSH,
+      finishFlush: constants.Z_SYNC_FLUSH
+    }
+
     // helper flag that helps us not yielding two subsequent undefined/disconnect messages
     let lastMessageWasUndefined = false
 
@@ -208,7 +202,10 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
       )
     }
 
-    await terminateWorker(worker, 500)
+    const underlyingWorker = worker.getUnderlyingWorker()
+    if (underlyingWorker !== undefined) {
+      await terminateWorker(underlyingWorker, 500)
+    }
   }
 }
 
@@ -330,6 +327,52 @@ function validateReplayNormalizedOptions(fromDate: Date, normalizers: MapperFact
   }
 }
 
+class ReliableWorker extends EventEmitter {
+  private _errorsCount = 0
+  private _worker: Worker | undefined = undefined
+
+  constructor(private readonly _payload: WorkerJobPayload) {
+    super()
+
+    this._initWorker()
+  }
+
+  private _initWorker() {
+    this._worker = new Worker(path.resolve(__dirname, 'worker.js'), {
+      workerData: this._payload
+    })
+
+    this._worker.on('message', (message: WorkerMessage) => {
+      this.emit('message', message)
+    })
+
+    this._worker.on('error', this._handleError)
+
+    this._worker.on('exit', (code) => {
+      debug('worker finished with code: %d', code)
+    })
+  }
+
+  private _handleError = async (err: Error) => {
+    debug('underlying worker error %o', err)
+
+    if (err.message.includes('HttpError') === false && this._errorsCount < 30) {
+      this._errorsCount++
+      const delayMS = Math.min(Math.pow(2, this._errorsCount) * 1000, 120 * 1000)
+      debug('re-init worker after: %d ms', delayMS)
+      await wait(delayMS)
+      // it was most likely unhandled socket hang up error, let's retry first with new worker and don't emit error right away
+      this._initWorker()
+    } else {
+      this.emit('error', err)
+    }
+  }
+
+  public getUnderlyingWorker() {
+    return this._worker
+  }
+}
+
 export type ReplayOptions<T extends Exchange, U extends boolean = false, Z extends boolean = false> = {
   readonly exchange: T
   readonly from: string

package/src/stream.ts

@@ -92,7 +92,7 @@ async function* _streamNormalized<T extends Exchange, U extends MapperFactory<T,
       for await (const message of normalizedMessages) {
         yield message
       }
-    } catch (error) {
+    } catch (error: any) {
       if (onError !== undefined) {
         onError(error)
       }