tardis-dev 13.5.0 → 13.6.2

package/src/consts.ts

@@ -44,7 +44,8 @@ export const EXCHANGES = [
   'huobi-dm-options',
   'star-atlas',
   'crypto-com',
-  'crypto-com-derivatives'
+  'crypto-com-derivatives',
+  'kucoin'
 ] as const
 
 const BINANCE_CHANNELS = ['trade', 'aggTrade', 'ticker', 'depth', 'depthSnapshot', 'bookTicker', 'recentTrades', 'borrowInterest'] as const
@@ -380,6 +381,8 @@ const CRYPTO_COM_CHANNELS = ['trade', 'book', 'ticker']
 
 const CRYPTO_COM_DERIVATIVES = ['trade', 'book', 'ticker', 'settlement', 'index', 'mark', 'funding']
 
+const KUCOIN_CHANNELS = ['market/ticker', 'market/snapshot', 'market/level2', 'market/match', 'market/level2Snapshot']
+
 export const EXCHANGE_CHANNELS_INFO = {
   bitmex: BITMEX_CHANNELS,
   coinbase: COINBASE_CHANNELS,
@@ -426,5 +429,6 @@ export const EXCHANGE_CHANNELS_INFO = {
   'huobi-dm-options': HUOBI_DM_OPTIONS_CHANNELS,
   mango: MANGO_CHANNELS,
   'crypto-com': CRYPTO_COM_CHANNELS,
-  'crypto-com-derivatives': CRYPTO_COM_DERIVATIVES
+  'crypto-com-derivatives': CRYPTO_COM_DERIVATIVES,
+  kucoin: KUCOIN_CHANNELS
 }

package/src/handy.ts

@@ -23,6 +23,10 @@ export function wait(delayMS: number) {
   })
 }
 
+export function getRandomString() {
+  return crypto.randomBytes(24).toString('hex')
+}
+
 export function formatDateToPath(date: Date) {
   const year = date.getUTCFullYear()
   const month = doubleDigit(date.getUTCMonth() + 1)

package/src/mappers/deribit.ts

@@ -151,7 +151,11 @@ export class DeribitOptionSummaryMapper implements Mapper<'deribit', OptionSumma
     }
 
     // options ticker has greeks
-    return channel.startsWith('ticker') && message.params.data.greeks !== undefined
+
+    return (
+      channel.startsWith('ticker') &&
+      (message.params.data.instrument_name.endsWith('-P') || message.params.data.instrument_name.endsWith('-C'))
+    )
   }
 
   *map(message: DeribitOptionTickerMessage, localTimestamp: Date) {

package/src/mappers/index.ts

@@ -69,6 +69,7 @@ import {
   HuobiTradesMapper
 } from './huobi'
 import { krakenBookChangeMapper, krakenBookTickerMapper, krakenTradesMapper } from './kraken'
+import { KucoinBookChangeMapper, KucoinBookTickerMapper, KucoinTradesMapper } from './kucoin'
 import { Mapper } from './mapper'
 import {
   OkexBookChangeMapper,
@@ -171,7 +172,8 @@ const tradesMappers = {
   mango: () => new SerumTradesMapper('mango'),
   'bybit-spot': () => new BybitSpotTradesMapper('bybit-spot'),
   'crypto-com': () => new CryptoComTradesMapper('crypto-com'),
-  'crypto-com-derivatives': () => new CryptoComTradesMapper('crypto-com-derivatives')
+  'crypto-com-derivatives': () => new CryptoComTradesMapper('crypto-com-derivatives'),
+  kucoin: () => new KucoinTradesMapper('kucoin')
 }
 
 const bookChangeMappers = {
@@ -241,7 +243,8 @@ const bookChangeMappers = {
   'star-atlas': () => new SerumBookChangeMapper('star-atlas'),
   mango: () => new SerumBookChangeMapper('mango'),
   'crypto-com': () => new CryptoComBookChangeMapper('crypto-com'),
-  'crypto-com-derivatives': () => new CryptoComBookChangeMapper('crypto-com-derivatives')
+  'crypto-com-derivatives': () => new CryptoComBookChangeMapper('crypto-com-derivatives'),
+  kucoin: (localTimestamp: Date) => new KucoinBookChangeMapper('kucoin', isRealTime(localTimestamp) === false)
 }
 
 const derivativeTickersMappers = {
@@ -349,7 +352,8 @@ const bookTickersMappers = {
   'gate-io-futures': () => new GateIOFuturesBookTickerMapper('gate-io-futures'),
   'bybit-spot': () => new BybitSpotBookTickerMapper('bybit-spot'),
   'crypto-com': () => new CryptoComBookTickerMapper('crypto-com'),
-  'crypto-com-derivatives': () => new CryptoComBookTickerMapper('crypto-com-derivatives')
+  'crypto-com-derivatives': () => new CryptoComBookTickerMapper('crypto-com-derivatives'),
+  kucoin: () => new KucoinBookTickerMapper('kucoin')
 }
 
 export const normalizeTrades = <T extends keyof typeof tradesMappers>(exchange: T, localTimestamp: Date): Mapper<T, Trade> => {

package/src/mappers/kucoin.ts

@@ -0,0 +1,315 @@
+import { debug } from '../debug'
+import { CircularBuffer, upperCaseSymbols } from '../handy'
+import { BookChange, Exchange, BookTicker, Trade, BookPriceLevel } from '../types'
+import { Mapper } from './mapper'
+
+export class KucoinTradesMapper implements Mapper<'kucoin', Trade> {
+  constructor(private readonly _exchange: Exchange) {}
+  canHandle(message: KucoinTradeMessage) {
+    return message.type === 'message' && message.topic.startsWith('/market/match')
+  }
+
+  getFilters(symbols?: string[]) {
+    symbols = upperCaseSymbols(symbols)
+
+    return [
+      {
+        channel: 'market/match',
+        symbols
+      } as const
+    ]
+  }
+
+  *map(message: KucoinTradeMessage, localTimestamp: Date): IterableIterator<Trade> {
+    const kucoinTrade = message.data
+
+    const timestamp = new Date(Number(kucoinTrade.time.slice(0, 13)))
+
+    timestamp.μs = Number(kucoinTrade.time.slice(13, 16))
+
+    yield {
+      type: 'trade',
+      symbol: kucoinTrade.symbol,
+      exchange: this._exchange,
+      id: kucoinTrade.tradeId,
+      price: Number(kucoinTrade.price),
+      amount: Number(kucoinTrade.size),
+      side: kucoinTrade.side === 'sell' ? 'sell' : 'buy',
+      timestamp,
+      localTimestamp
+    }
+  }
+}
+
+export class KucoinBookChangeMapper implements Mapper<'kucoin', BookChange> {
+  protected readonly symbolToDepthInfoMapping: {
+    [key: string]: LocalDepthInfo
+  } = {}
+
+  constructor(protected readonly _exchange: Exchange, private readonly ignoreBookSnapshotOverlapError: boolean) {}
+
+  canHandle(message: KucoinLevel2SnapshotMessage | KucoinLevel2UpdateMessage) {
+    return message.type === 'message' && message.topic.startsWith('/market/level2')
+  }
+
+  getFilters(symbols?: string[]) {
+    symbols = upperCaseSymbols(symbols)
+    return [
+      {
+        channel: 'market/level2',
+        symbols
+      } as const,
+      {
+        channel: 'market/level2Snapshot',
+        symbols
+      } as const
+    ]
+  }
+
+  *map(message: KucoinLevel2SnapshotMessage | KucoinLevel2UpdateMessage, localTimestamp: Date) {
+    const symbol = message.topic.split(':')[1]
+
+    if (this.symbolToDepthInfoMapping[symbol] === undefined) {
+      this.symbolToDepthInfoMapping[symbol] = {
+        bufferedUpdates: new CircularBuffer<KucoinLevel2UpdateMessage>(2000)
+      }
+    }
+
+    const symbolDepthInfo = this.symbolToDepthInfoMapping[symbol]
+    const snapshotAlreadyProcessed = symbolDepthInfo.snapshotProcessed
+
+    // first check if received message is snapshot and process it as such if it is
+    if (message.subject === 'trade.l2Snapshot') {
+      // if we've already received 'manual' snapshot, ignore if there is another one
+      if (snapshotAlreadyProcessed) {
+        return
+      }
+      // produce snapshot book_change
+      const kucoinSnapshotData = message.data
+
+      //  mark given symbol depth info that has snapshot processed
+      symbolDepthInfo.lastUpdateId = Number(kucoinSnapshotData.sequence)
+      symbolDepthInfo.snapshotProcessed = true
+
+      // if there were any depth updates buffered, let's proccess those by adding to or updating the initial snapshot
+      for (const update of symbolDepthInfo.bufferedUpdates.items()) {
+        const bookChange = this.mapBookDepthUpdate(update, localTimestamp)
+        if (bookChange !== undefined) {
+          for (const bid of update.data.changes.bids) {
+            if (bid[0] == '0') {
+              continue
+            }
+            const matchingBid = kucoinSnapshotData.bids.find((b) => b[0] === bid[0])
+            if (matchingBid !== undefined) {
+              matchingBid[1] = bid[1]
+            } else {
+              kucoinSnapshotData.bids.push([bid[0], bid[1]])
+            }
+          }
+
+          for (const ask of update.data.changes.asks) {
+            if (ask[0] == '0') {
+              continue
+            }
+
+            const matchingAsk = kucoinSnapshotData.asks.find((a) => a[0] === ask[0])
+            if (matchingAsk !== undefined) {
+              matchingAsk[1] = ask[1]
+            } else {
+              kucoinSnapshotData.asks.push([ask[0], ask[1]])
+            }
+          }
+        }
+      }
+
+      // remove all buffered updates
+      symbolDepthInfo.bufferedUpdates.clear()
+
+      const bookChange: BookChange = {
+        type: 'book_change',
+        symbol,
+        exchange: this._exchange,
+        isSnapshot: true,
+        bids: kucoinSnapshotData.bids.map(this.mapBookLevel),
+        asks: kucoinSnapshotData.asks.map(this.mapBookLevel),
+        timestamp: localTimestamp,
+        localTimestamp
+      }
+
+      yield bookChange
+    } else if (snapshotAlreadyProcessed) {
+      // snapshot was already processed let's map the message as normal book_change
+      const bookChange = this.mapBookDepthUpdate(message, localTimestamp)
+      if (bookChange !== undefined) {
+        yield bookChange
+      }
+    } else {
+      symbolDepthInfo.bufferedUpdates.append(message)
+    }
+  }
+
+  protected mapBookDepthUpdate(l2UpdateMessage: KucoinLevel2UpdateMessage, localTimestamp: Date): BookChange | undefined {
+    // we can safely assume here that depthContext and lastUpdateId aren't null here as this is method only works
+    // when we've already processed the snapshot
+    const depthContext = this.symbolToDepthInfoMapping[l2UpdateMessage.data.symbol]!
+    const lastUpdateId = depthContext.lastUpdateId!
+
+    // Drop any event where sequenceEnd is <= lastUpdateId in the snapshot
+    if (l2UpdateMessage.data.sequenceEnd <= lastUpdateId) {
+      return
+    }
+
+    // The first processed event should have sequenceStart <= lastUpdateId+1 AND sequenceEnd >= lastUpdateId+1.
+    if (!depthContext.validatedFirstUpdate) {
+      // if there is new instrument added it can have empty book at first and that's normal
+      const bookSnapshotIsEmpty = lastUpdateId == -1 || lastUpdateId == 0
+
+      if (
+        (l2UpdateMessage.data.sequenceStart <= lastUpdateId + 1 && l2UpdateMessage.data.sequenceEnd >= lastUpdateId + 1) ||
+        bookSnapshotIsEmpty
+      ) {
+        depthContext.validatedFirstUpdate = true
+      } else {
+        const message = `Book depth snapshot has no overlap with first update, update ${JSON.stringify(
+          l2UpdateMessage
+        )}, lastUpdateId: ${lastUpdateId}, exchange ${this._exchange}`
+        if (this.ignoreBookSnapshotOverlapError) {
+          depthContext.validatedFirstUpdate = true
+          debug(message)
+        } else {
+          throw new Error(message)
+        }
+      }
+    }
+    const bids = l2UpdateMessage.data.changes.bids.map(this.mapBookLevel).filter(this.nonZeroLevels)
+    const asks = l2UpdateMessage.data.changes.asks.map(this.mapBookLevel).filter(this.nonZeroLevels)
+
+    if (bids.length === 0 && asks.length === 0) {
+      return
+    }
+
+    return {
+      type: 'book_change',
+      symbol: l2UpdateMessage.data.symbol,
+      exchange: this._exchange,
+      isSnapshot: false,
+      bids,
+      asks,
+      timestamp: localTimestamp,
+      localTimestamp: localTimestamp
+    }
+  }
+
+  private mapBookLevel(level: [string, string, string?]) {
+    const price = Number(level[0])
+    const amount = Number(level[1])
+    return { price, amount }
+  }
+
+  private nonZeroLevels(level: BookPriceLevel) {
+    return level.price > 0
+  }
+}
+
+export class KucoinBookTickerMapper implements Mapper<'kucoin', BookTicker> {
+  constructor(protected readonly _exchange: Exchange) {}
+
+  canHandle(message: KucoinTickerMessage) {
+    return message.type === 'message' && message.topic.startsWith('/market/ticker')
+  }
+
+  getFilters(symbols?: string[]) {
+    symbols = upperCaseSymbols(symbols)
+    return [
+      {
+        channel: 'market/ticker',
+        symbols
+      } as const
+    ]
+  }
+
+  *map(message: KucoinTickerMessage, localTimestamp: Date) {
+    const symbol = message.topic.split(':')[1]
+
+    const bookTicker: BookTicker = {
+      type: 'book_ticker',
+      symbol,
+      exchange: this._exchange,
+      askAmount: message.data.bestAskSize !== undefined && message.data.bestAskSize !== null ? Number(message.data.bestAskSize) : undefined,
+      askPrice: message.data.bestAsk !== undefined && message.data.bestAsk !== null ? Number(message.data.bestAsk) : undefined,
+      bidPrice: message.data.bestBid !== undefined && message.data.bestBid !== null ? Number(message.data.bestBid) : undefined,
+      bidAmount: message.data.bestBidSize !== undefined && message.data.bestBidSize !== null ? Number(message.data.bestBidSize) : undefined,
+      timestamp: new Date(message.data.time),
+      localTimestamp: localTimestamp
+    }
+
+    yield bookTicker
+  }
+}
+
+type KucoinTickerMessage = {
+  type: 'message'
+  topic: '/market/ticker:ADA-USDT'
+  subject: 'trade.ticker'
+  data: {
+    bestAsk: '0.549931'
+    bestAskSize: '966.4756'
+    bestBid: '0.549824'
+    bestBidSize: '1050'
+    price: '0.549825'
+    sequence: '1623526404099'
+    size: '1'
+    time: 1660608019871
+  }
+}
+
+type KucoinTradeMessage = {
+  type: 'message'
+  topic: '/market/match:BTC-USDT'
+  subject: 'trade.l3match'
+  data: {
+    symbol: 'BTC-USDT'
+    side: 'sell'
+    type: 'match'
+    makerOrderId: '62fadde41add68000167fb58'
+    sequence: '1636276321894'
+    size: '0.00001255'
+    price: '24093.9'
+    takerOrderId: '62faddfff0476c0001c86c71'
+    time: '1660608000026914990'
+    tradeId: '62fade002e113d292303a18b'
+  }
+}
+
+type LocalDepthInfo = {
+  bufferedUpdates: CircularBuffer<KucoinLevel2UpdateMessage>
+  snapshotProcessed?: boolean
+  lastUpdateId?: number
+  validatedFirstUpdate?: boolean
+}
+
+type KucoinLevel2SnapshotMessage = {
+  type: 'message'
+  generated: true
+  topic: '/market/level2Snapshot:BTC-USDT'
+  subject: 'trade.l2Snapshot'
+  code: '200000'
+  data: {
+    time: 1660608003710
+    sequence: '1636276324355'
+    bids: [string, string][]
+    asks: [string, string][]
+  }
+}
+
+type KucoinLevel2UpdateMessage = {
+  type: 'message'
+  topic: '/market/level2:BTC-USDT'
+  subject: 'trade.l2update'
+  data: {
+    sequenceStart: 1636276324710
+    symbol: 'BTC-USDT'
+    changes: { asks: [string, string, string][]; bids: [string, string, string][] }
+    sequenceEnd: 1636276324710
+  }
+}

package/src/realtimefeeds/binance.ts

@@ -39,7 +39,7 @@ class BinanceFuturesOpenInterestClient extends PoolingClientBase {
 
   protected async poolDataToStream(outputStream: Writable) {
     for (const instruments of batch(this._instruments, 10)) {
-      await Promise.all(
+      await Promise.allSettled(
         instruments.map(async (instrument) => {
           if (outputStream.destroyed) {
             return

package/src/realtimefeeds/ftx.ts

@@ -85,7 +85,7 @@ class FTXInstrumentInfoClient extends PoolingClientBase {
 
   protected async poolDataToStream(outputStream: Writable) {
     for (const instruments of batch(this._instruments, 10)) {
-      await Promise.all(
+      await Promise.allSettled(
         instruments.map(async (instrument) => {
           if (outputStream.destroyed) {
             return

package/src/realtimefeeds/huobi.ts

@@ -241,7 +241,7 @@ class HuobiOpenInterestClient extends PoolingClientBase {
 
   protected async poolDataToStream(outputStream: Writable) {
     for (const instruments of batch(this._instruments, 10)) {
-      await Promise.all(
+      await Promise.allSettled(
         instruments.map(async (instrument) => {
           if (outputStream.destroyed) {
             return
@@ -311,7 +311,7 @@ class HuobiOptionsIndexClient extends PoolingClientBase {
 
   protected async poolDataToStream(outputStream: Writable) {
     for (const instruments of batch(this._instruments, 10)) {
-      await Promise.all(
+      await Promise.allSettled(
         instruments.map(async (instrument) => {
           if (outputStream.destroyed) {
             return

package/src/realtimefeeds/index.ts

@@ -44,6 +44,7 @@ import { MangoRealTimeFeed } from './mango'
 import { BybitSpotRealTimeFeed } from './bybitspot'
 import { CryptoComRealTimeFeed } from './cryptocom'
 import { CryptoComDerivativesRealTimeFeed } from './cryptocomderivatives'
+import { KucoinRealTimeFeed } from './kucoin'
 
 export * from './realtimefeed'
 
@@ -95,7 +96,8 @@ const realTimeFeedsMap: {
   mango: MangoRealTimeFeed,
   'bybit-spot': BybitSpotRealTimeFeed,
   'crypto-com': CryptoComRealTimeFeed,
-  'crypto-com-derivatives': CryptoComDerivativesRealTimeFeed
+  'crypto-com-derivatives': CryptoComDerivativesRealTimeFeed,
+  kucoin: KucoinRealTimeFeed
 }
 
 export function getRealTimeFeedFactory(exchange: Exchange): RealTimeFeed {

package/src/realtimefeeds/kucoin.ts

@@ -0,0 +1,77 @@
+import { httpClient, getRandomString, wait } from '../handy'
+import { Filter } from '../types'
+import { RealTimeFeedBase } from './realtimefeed'
+
+export class KucoinRealTimeFeed extends RealTimeFeedBase {
+  protected wssURL = ''
+  private _httpURL = 'https://api.kucoin.com/api'
+
+  protected async getWebSocketUrl() {
+    const response = (await httpClient.post(`${this._httpURL}/v1/bullet-public`, { retry: 3, timeout: 10000 }).json()) as any
+
+    return `${response.data.instanceServers[0].endpoint}?token=${response.data.token}&connectId=${getRandomString()}`
+  }
+
+  protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
+    return filters
+      .filter((f) => f.channel !== 'market/level2Snapshot')
+      .map((filter) => {
+        if (!filter.symbols || filter.symbols.length === 0) {
+          throw new Error('KucoinRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
+        }
+
+        return {
+          id: getRandomString(),
+          type: 'subscribe',
+          topic: `/${filter.channel}:${filter.symbols.join(',')}`,
+          privateChannel: false,
+          response: true
+        }
+      })
+  }
+
+  protected async provideManualSnapshots(filters: Filter<string>[], shouldCancel: () => boolean) {
+    const depthSnapshotFilter = filters.find((f) => f.channel === 'market/level2Snapshot')
+    if (!depthSnapshotFilter) {
+      return
+    }
+
+    this.debug('requesting manual snapshots for: %s', depthSnapshotFilter.symbols)
+    for (let symbol of depthSnapshotFilter.symbols!) {
+      if (shouldCancel()) {
+        return
+      }
+
+      const depthSnapshotResponse = (await httpClient
+        .get(`${this._httpURL}/v1/market/orderbook/level2_100?symbol=${symbol}`, { timeout: 10000 })
+        .json()) as any
+
+      const snapshot = {
+        type: 'message',
+        generated: true,
+        topic: `/market/level2Snapshot:${symbol}`,
+        subject: 'trade.l2Snapshot',
+        ...depthSnapshotResponse
+      }
+
+      this.manualSnapshotsBuffer.push(snapshot)
+    }
+
+    this.debug('requested manual snapshots successfully for: %s ', depthSnapshotFilter.symbols)
+  }
+
+  protected messageIsError(message: any): boolean {
+    return message.type === 'error'
+  }
+
+  protected sendCustomPing = () => {
+    this.send({
+      id: new Date().valueOf().toString(),
+      type: 'ping'
+    })
+  }
+
+  protected messageIsHeartbeat(msg: any) {
+    return msg.type === 'pong'
+  }
+}

package/src/realtimefeeds/realtimefeed.ts

@@ -58,6 +58,13 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
     }
   }
 
+  protected async getWebSocketUrl() {
+    const wssUrlOverride = process.env[`WSS_URL_${this._exchange.toUpperCase()}`]
+    const finalWssUrl = wssUrlOverride !== undefined ? wssUrlOverride : this.wssURL
+
+    return finalWssUrl
+  }
+
   private async *_stream() {
     let staleConnectionTimerId
     let pingTimerId
@@ -66,9 +73,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
     while (true) {
       try {
         const subscribeMessages = this.mapToSubscribeMessages(this._filters)
-
-        const wssUrlOverride = process.env[`WSS_URL_${this._exchange.toUpperCase()}`]
-        const finalWssUrl = wssUrlOverride !== undefined ? wssUrlOverride : this.wssURL
+        const finalWssUrl = await this.getWebSocketUrl()
 
         this.debug('(connection id: %d) estabilishing connection to %s', this._connectionId, finalWssUrl)