tardis-dev 16.5.1 → 16.6.0

package/src/mappers/kraken.ts

@@ -1,10 +1,19 @@
 import { asNonZeroNumberOrUndefined, upperCaseSymbols } from '../handy.ts'
 import { BookChange, BookTicker, Trade } from '../types.ts'
 import { Mapper } from './mapper.ts'
+import { exchangeMappers } from './registry.ts'
 
 // https://www.kraken.com/features/websocket-api
 
-export const krakenTradesMapper: Mapper<'kraken', Trade> = {
+export const krakenMappers = exchangeMappers({
+  kraken: {
+    trades: () => krakenTradesMapper,
+    bookChanges: () => krakenBookChangeMapper,
+    bookTickers: () => krakenBookTickerMapper
+  }
+})
+
+const krakenTradesMapper: Mapper<'kraken', Trade> = {
   canHandle(message: KrakenTrades) {
     if (!Array.isArray(message)) {
       return false
@@ -64,7 +73,7 @@ const getLatestTimestamp = (bids: KrakenBookLevel[], asks: KrakenBookLevel[]): D
   return timestamp
 }
 
-export const krakenBookChangeMapper: Mapper<'kraken', BookChange> = {
+const krakenBookChangeMapper: Mapper<'kraken', BookChange> = {
   canHandle(message: KrakenBookSnapshot | KrakenBookUpdate) {
     if (!Array.isArray(message)) {
       return false
@@ -122,7 +131,7 @@ export const krakenBookChangeMapper: Mapper<'kraken', BookChange> = {
   }
 }
 
-export const krakenBookTickerMapper: Mapper<'kraken', BookTicker> = {
+const krakenBookTickerMapper: Mapper<'kraken', BookTicker> = {
   canHandle(message: KrakenSpread) {
     if (!Array.isArray(message)) {
       return false

package/src/mappers/kucoin.ts

@@ -2,8 +2,18 @@ import { debug } from '../debug.ts'
 import { CircularBuffer, upperCaseSymbols } from '../handy.ts'
 import { BookChange, Exchange, BookTicker, Trade, BookPriceLevel } from '../types.ts'
 import { Mapper } from './mapper.ts'
+import { exchangeMappers, isRealTime } from './registry.ts'
+
+export const kucoinMappers = exchangeMappers({
+  kucoin: {
+    trades: () => new KucoinTradesMapper('kucoin'),
+    bookChanges: (localTimestamp) =>
+      new KucoinBookChangeMapper('kucoin', { ignoreBookSnapshotOverlapError: isRealTime(localTimestamp) === false }),
+    bookTickers: () => new KucoinBookTickerMapper('kucoin')
+  }
+})
 
-export class KucoinTradesMapper implements Mapper<'kucoin', Trade> {
+class KucoinTradesMapper implements Mapper<'kucoin', Trade> {
   constructor(private readonly _exchange: Exchange) {}
   canHandle(message: KucoinTradeMessage) {
     return message.type === 'message' && message.topic.startsWith('/market/match')
@@ -41,15 +51,19 @@ export class KucoinTradesMapper implements Mapper<'kucoin', Trade> {
   }
 }
 
-export class KucoinBookChangeMapper implements Mapper<'kucoin', BookChange> {
+class KucoinBookChangeMapper implements Mapper<'kucoin', BookChange> {
   protected readonly symbolToDepthInfoMapping: {
     [key: string]: LocalDepthInfo
   } = {}
 
   constructor(
     protected readonly _exchange: Exchange,
-    private readonly ignoreBookSnapshotOverlapError: boolean
-  ) {}
+    { ignoreBookSnapshotOverlapError }: { ignoreBookSnapshotOverlapError: boolean }
+  ) {
+    this.ignoreBookSnapshotOverlapError = ignoreBookSnapshotOverlapError
+  }
+
+  private readonly ignoreBookSnapshotOverlapError: boolean
 
   canHandle(message: KucoinLevel2SnapshotMessage | KucoinLevel2UpdateMessage) {
     return message.type === 'message' && message.topic.startsWith('/market/level2')
@@ -221,7 +235,7 @@ export class KucoinBookChangeMapper implements Mapper<'kucoin', BookChange> {
   }
 }
 
-export class KucoinBookTickerMapper implements Mapper<'kucoin', BookTicker> {
+class KucoinBookTickerMapper implements Mapper<'kucoin', BookTicker> {
   constructor(protected readonly _exchange: Exchange) {}
 
   canHandle(message: KucoinTickerMessage) {

package/src/mappers/kucoinfutures.ts

@@ -2,8 +2,19 @@ import { debug } from '../debug.ts'
 import { asNonZeroNumberOrUndefined, CircularBuffer, upperCaseSymbols } from '../handy.ts'
 import { BookChange, BookTicker, DerivativeTicker, Trade } from '../types.ts'
 import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
+import { exchangeMappers, isRealTime } from './registry.ts'
+
+export const kucoinFuturesMappers = exchangeMappers({
+  'kucoin-futures': {
+    trades: () => new KucoinFuturesTradesMapper(),
+    bookChanges: (localTimestamp) =>
+      new KucoinFuturesBookChangeMapper({ ignoreBookSnapshotOverlapError: isRealTime(localTimestamp) === false }),
+    derivativeTickers: () => new KucoinFuturesDerivativeTickerMapper(),
+    bookTickers: () => new KucoinFuturesBookTickerMapper()
+  }
+})
 
-export class KucoinFuturesTradesMapper implements Mapper<'kucoin-futures', Trade> {
+class KucoinFuturesTradesMapper implements Mapper<'kucoin-futures', Trade> {
   canHandle(message: KucoinFuturesTradeMessage) {
     return message.type === 'message' && message.topic.startsWith('/contractMarket/execution')
   }
@@ -38,12 +49,16 @@ export class KucoinFuturesTradesMapper implements Mapper<'kucoin-futures', Trade
   }
 }
 
-export class KucoinFuturesBookChangeMapper implements Mapper<'kucoin-futures', BookChange> {
+class KucoinFuturesBookChangeMapper implements Mapper<'kucoin-futures', BookChange> {
   protected readonly symbolToDepthInfoMapping: {
     [key: string]: LocalDepthInfo
   } = {}
 
-  constructor(private readonly ignoreBookSnapshotOverlapError: boolean) {}
+  private readonly ignoreBookSnapshotOverlapError: boolean
+
+  constructor({ ignoreBookSnapshotOverlapError }: { ignoreBookSnapshotOverlapError: boolean }) {
+    this.ignoreBookSnapshotOverlapError = ignoreBookSnapshotOverlapError
+  }
 
   canHandle(message: KucoinFuturesLevel2SnapshotMessage | KucoinFuturesLevel2UpdateMessage) {
     return message.type === 'message' && message.topic.startsWith('/contractMarket/level2')
@@ -229,7 +244,7 @@ export class KucoinFuturesBookChangeMapper implements Mapper<'kucoin-futures', B
   }
 }
 
-export class KucoinFuturesBookTickerMapper implements Mapper<'kucoin-futures', BookTicker> {
+class KucoinFuturesBookTickerMapper implements Mapper<'kucoin-futures', BookTicker> {
   canHandle(message: KucoinFuturesTickerMessage) {
     return message.type === 'message' && message.topic.startsWith('/contractMarket/tickerV2')
   }
@@ -267,7 +282,7 @@ export class KucoinFuturesBookTickerMapper implements Mapper<'kucoin-futures', B
   }
 }
 
-export class KucoinFuturesDerivativeTickerMapper implements Mapper<'kucoin-futures', DerivativeTicker> {
+class KucoinFuturesDerivativeTickerMapper implements Mapper<'kucoin-futures', DerivativeTicker> {
   private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
   private readonly _lastPrices = new Map<string, number>()
   private readonly _openInterests = new Map<string, number>()

package/src/mappers/lighter.ts

@@ -1,6 +1,17 @@
 import { asNonZeroNumberOrUndefined } from '../handy.ts'
 import { BookChange, BookTicker, DerivativeTicker, Liquidation, Trade } from '../types.ts'
 import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
+import { exchangeMappers } from './registry.ts'
+
+export const lighterMappers = exchangeMappers({
+  lighter: {
+    trades: () => new LighterTradesMapper(),
+    bookChanges: () => new LighterBookChangeMapper(),
+    derivativeTickers: () => new LighterDerivativeTickerMapper(),
+    liquidations: () => new LighterLiquidationMapper(),
+    bookTickers: () => new LighterBookTickerMapper()
+  }
+})
 
 function parseChannelMarketId(channel: string): string | undefined {
   const colonIndex = channel.indexOf(':')
@@ -14,7 +25,7 @@ function parseChannelMarketId(channel: string): string | undefined {
   return suffix
 }
 
-export class LighterTradesMapper implements Mapper<'lighter', Trade> {
+class LighterTradesMapper implements Mapper<'lighter', Trade> {
   canHandle(message: LighterTradeMessage) {
     return message.type === 'update/trade'
   }
@@ -45,7 +56,7 @@ export class LighterTradesMapper implements Mapper<'lighter', Trade> {
   }
 }
 
-export class LighterLiquidationMapper implements Mapper<'lighter', Liquidation> {
+class LighterLiquidationMapper implements Mapper<'lighter', Liquidation> {
   canHandle(message: LighterTradeMessage) {
     return message.type === 'update/trade'
   }
@@ -84,7 +95,7 @@ export class LighterLiquidationMapper implements Mapper<'lighter', Liquidation>
   }
 }
 
-export class LighterBookChangeMapper implements Mapper<'lighter', BookChange> {
+class LighterBookChangeMapper implements Mapper<'lighter', BookChange> {
   canHandle(message: LighterOrderBookMessage) {
     return message.type === 'subscribed/order_book' || message.type === 'update/order_book'
   }
@@ -122,7 +133,7 @@ export class LighterBookChangeMapper implements Mapper<'lighter', BookChange> {
   }
 }
 
-export class LighterBookTickerMapper implements Mapper<'lighter', BookTicker> {
+class LighterBookTickerMapper implements Mapper<'lighter', BookTicker> {
   canHandle(message: LighterTickerMessage) {
     return message.type === 'update/ticker'
   }
@@ -154,7 +165,7 @@ export class LighterBookTickerMapper implements Mapper<'lighter', BookTicker> {
   }
 }
 
-export class LighterDerivativeTickerMapper implements Mapper<'lighter', DerivativeTicker> {
+class LighterDerivativeTickerMapper implements Mapper<'lighter', DerivativeTicker> {
   private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
 
   canHandle(message: LighterMarketStatsMessage) {

package/src/mappers/mapper.ts

@@ -8,7 +8,9 @@ export type Mapper<T extends Exchange, U extends NormalizedData> = {
   getFilters: (symbols?: string[]) => FilterForExchange[T][]
 }
 
-export type MapperFactory<T extends Exchange, U extends NormalizedData> = (exchange: T, localTimestamp: Date) => Mapper<T, U>
+export type MapperFactory<T extends Exchange, U extends NormalizedData> = ((exchange: T, localTimestamp: Date) => Mapper<T, U>) & {
+  getSwitchDates?: (exchange: T) => readonly Date[]
+}
 
 type Writeable<T> = { -readonly [P in keyof T]: T[P] }
 

package/src/mappers/mexc.ts

@@ -0,0 +1,324 @@
+import { debug } from '../debug.ts'
+import { asNumberOrUndefined, CircularBuffer, upperCaseSymbols } from '../handy.ts'
+import { BookChange, BookPriceLevel, BookTicker, Trade } from '../types.ts'
+import { Mapper } from './mapper.ts'
+import { exchangeMappers, isRealTime } from './registry.ts'
+
+export const mexcMappers = exchangeMappers({
+  mexc: {
+    trades: () => new MexcTradesMapper(),
+    bookChanges: (localTimestamp) => new MexcBookChangeMapper({ ignoreBookSnapshotOverlapError: isRealTime(localTimestamp) === false }),
+    bookTickers: () => new MexcBookTickerMapper()
+  }
+})
+
+export class MexcTradesMapper implements Mapper<'mexc', Trade> {
+  private readonly channel = 'spot@public.aggre.deals.v3.api.pb@10ms'
+
+  canHandle(message: MexcTradeMessage | MexcControlMessage) {
+    return message.channel?.startsWith(`${this.channel}@`) === true && 'publicAggreDeals' in message
+  }
+
+  getFilters(symbols?: string[]) {
+    return [{ channel: this.channel, symbols: upperCaseSymbols(symbols) } as const]
+  }
+
+  *map(message: MexcTradeMessage, localTimestamp: Date): IterableIterator<Trade> {
+    for (const trade of message.publicAggreDeals.deals) {
+      yield {
+        type: 'trade',
+        symbol: message.symbol,
+        exchange: 'mexc',
+        id: trade.tradeId,
+        price: Number(trade.price),
+        amount: Number(trade.quantity),
+        side: trade.tradeType === MexcTradeType.Buy ? 'buy' : 'sell',
+        timestamp: new Date(Number(trade.time)),
+        localTimestamp
+      }
+    }
+  }
+}
+
+export class MexcBookChangeMapper implements Mapper<'mexc', BookChange> {
+  private readonly channel = 'spot@public.aggre.depth.v3.api.pb@10ms'
+  private readonly snapshotChannel = 'spot@public.aggre.depth.snapshot.v3.api.pb@10ms'
+  private readonly symbolDepthInfo: Record<string, MexcDepthInfo> = {}
+  private readonly ignoreBookSnapshotOverlapError: boolean
+
+  constructor({ ignoreBookSnapshotOverlapError }: { ignoreBookSnapshotOverlapError: boolean }) {
+    this.ignoreBookSnapshotOverlapError = ignoreBookSnapshotOverlapError
+  }
+
+  canHandle(message: MexcDepthSnapshotMessage | MexcDepthUpdateMessage) {
+    return message.generated === true
+      ? message.channel?.startsWith(`${this.snapshotChannel}@`) === true && 'publicAggreDepthsSnapshot' in message
+      : message.channel?.startsWith(`${this.channel}@`) === true && 'publicAggreDepths' in message
+  }
+
+  getFilters(symbols?: string[]) {
+    const normalizedSymbols = upperCaseSymbols(symbols)
+
+    return [
+      { channel: this.channel, symbols: normalizedSymbols } as const,
+      { channel: this.snapshotChannel, symbols: normalizedSymbols } as const
+    ]
+  }
+
+  *map(message: MexcDepthSnapshotMessage | MexcDepthUpdateMessage, localTimestamp: Date): IterableIterator<BookChange> {
+    const depthInfo = this.getDepthInfoFor(message.symbol)
+    if (message.generated === true) {
+      if (depthInfo.snapshotEmitted) {
+        return
+      }
+
+      let currentBookVersion = Number(message.publicAggreDepthsSnapshot.lastUpdateId)
+      const bids = message.publicAggreDepthsSnapshot.bids.map(this.mapDepthSnapshotLevel)
+      const asks = message.publicAggreDepthsSnapshot.asks.map(this.mapDepthSnapshotLevel)
+
+      for (const update of depthInfo.updates.items()) {
+        const fromVersion = Number(update.publicAggreDepths.fromVersion)
+        const toVersion = Number(update.publicAggreDepths.toVersion)
+        if (currentBookVersion >= toVersion) {
+          continue
+        }
+
+        if (!depthInfo.isContinuityValidated) {
+          if (fromVersion > currentBookVersion + 1 || toVersion < currentBookVersion + 1) {
+            const message = `MEXC depth snapshot has no overlap with first update, update ${JSON.stringify(update)}, currentBookVersion: ${currentBookVersion}`
+            if (this.ignoreBookSnapshotOverlapError) {
+              depthInfo.isContinuityValidated = true
+              debug(message)
+            } else {
+              throw new Error(message)
+            }
+          } else {
+            depthInfo.isContinuityValidated = true
+          }
+        }
+
+        for (const bid of update.publicAggreDepths.bids ?? []) {
+          this.applyLevel(bids, this.mapBookLevel(bid))
+        }
+        for (const ask of update.publicAggreDepths.asks ?? []) {
+          this.applyLevel(asks, this.mapBookLevel(ask))
+        }
+        currentBookVersion = toVersion
+      }
+
+      depthInfo.updates.clear()
+      depthInfo.currentBookVersion = currentBookVersion
+      depthInfo.snapshotEmitted = true
+
+      yield {
+        type: 'book_change',
+        symbol: message.symbol,
+        exchange: 'mexc',
+        isSnapshot: true,
+        bids,
+        asks,
+        timestamp: new Date(Number(message.publicAggreDepthsSnapshot.timestamp)),
+        localTimestamp
+      }
+
+      return
+    }
+
+    if (!depthInfo.snapshotEmitted) {
+      depthInfo.updates.append(message)
+      return
+    }
+
+    const fromVersion = Number(message.publicAggreDepths.fromVersion)
+    const toVersion = Number(message.publicAggreDepths.toVersion)
+    if (toVersion <= depthInfo.currentBookVersion!) {
+      return
+    }
+
+    if (!depthInfo.isContinuityValidated) {
+      if (fromVersion > depthInfo.currentBookVersion! + 1 || toVersion < depthInfo.currentBookVersion! + 1) {
+        const errorMessage = `MEXC depth snapshot has no overlap with first update, update ${JSON.stringify(message)}, currentBookVersion: ${depthInfo.currentBookVersion}`
+        if (this.ignoreBookSnapshotOverlapError) {
+          depthInfo.isContinuityValidated = true
+          debug(errorMessage)
+        } else {
+          throw new Error(errorMessage)
+        }
+      } else {
+        depthInfo.isContinuityValidated = true
+      }
+    }
+
+    depthInfo.currentBookVersion = toVersion
+
+    yield {
+      type: 'book_change',
+      symbol: message.symbol,
+      exchange: 'mexc',
+      isSnapshot: false,
+      bids: (message.publicAggreDepths.bids ?? []).map(this.mapBookLevel),
+      asks: (message.publicAggreDepths.asks ?? []).map(this.mapBookLevel),
+      timestamp: new Date(Number(message.sendTime)),
+      localTimestamp
+    }
+  }
+
+  private applyLevel(bookSide: BookPriceLevel[], levelUpdate: BookPriceLevel) {
+    const existingIndex = bookSide.findIndex((level) => level.price === levelUpdate.price)
+    if (levelUpdate.amount === 0) {
+      if (existingIndex !== -1) {
+        bookSide.splice(existingIndex, 1)
+      }
+      return
+    }
+
+    if (existingIndex === -1) {
+      bookSide.push(levelUpdate)
+    } else {
+      bookSide[existingIndex] = levelUpdate
+    }
+  }
+
+  private mapBookLevel(level: MexcPriceLevel) {
+    return {
+      price: Number(level.price),
+      amount: Number(level.quantity)
+    }
+  }
+
+  private mapDepthSnapshotLevel([price, quantity]: MexcDepthSnapshotLevel) {
+    return {
+      price: Number(price),
+      amount: Number(quantity)
+    }
+  }
+
+  private getDepthInfoFor(symbol: string) {
+    if (this.symbolDepthInfo[symbol] === undefined) {
+      this.symbolDepthInfo[symbol] = { updates: new CircularBuffer<MexcDepthUpdateMessage>(2000) }
+    }
+
+    return this.symbolDepthInfo[symbol]
+  }
+}
+
+export class MexcBookTickerMapper implements Mapper<'mexc', BookTicker> {
+  private readonly channel = 'spot@public.aggre.bookTicker.v3.api.pb@10ms'
+
+  canHandle(message: MexcBookTickerMessage | MexcControlMessage) {
+    return message.channel?.startsWith(`${this.channel}@`) === true && 'publicAggreBookTicker' in message
+  }
+
+  getFilters(symbols?: string[]) {
+    return [{ channel: this.channel, symbols: upperCaseSymbols(symbols) } as const]
+  }
+
+  *map(message: MexcBookTickerMessage, localTimestamp: Date): IterableIterator<BookTicker> {
+    yield {
+      type: 'book_ticker',
+      symbol: message.symbol,
+      exchange: 'mexc',
+      askAmount: asNumberOrUndefined(message.publicAggreBookTicker.askQuantity),
+      askPrice: asNumberOrUndefined(message.publicAggreBookTicker.askPrice),
+      bidAmount: asNumberOrUndefined(message.publicAggreBookTicker.bidQuantity),
+      bidPrice: asNumberOrUndefined(message.publicAggreBookTicker.bidPrice),
+      timestamp: new Date(Number(message.sendTime)),
+      localTimestamp
+    }
+  }
+}
+
+type MexcControlMessage = {
+  channel?: undefined
+  id?: number
+  code?: number
+  msg?: string
+}
+
+type MexcMappedChannel =
+  | 'spot@public.aggre.deals.v3.api.pb@10ms'
+  | 'spot@public.aggre.depth.v3.api.pb@10ms'
+  | 'spot@public.aggre.depth.snapshot.v3.api.pb@10ms'
+  | 'spot@public.aggre.bookTicker.v3.api.pb@10ms'
+type MexcChannelWithSymbol<TChannel extends MexcMappedChannel> = `${TChannel}@${string}`
+
+type MexcProtobufMessage<TChannel extends MexcMappedChannel> = {
+  channel: MexcChannelWithSymbol<TChannel>
+  symbol: string
+  symbolId?: string
+  createTime?: string | number
+  sendTime: string | number
+}
+
+type MexcTradeMessage = MexcProtobufMessage<'spot@public.aggre.deals.v3.api.pb@10ms'> & {
+  publicAggreDeals: {
+    eventType: 'spot@public.aggre.deals.v3.api.pb@10ms'
+    deals: MexcTrade[]
+  }
+}
+
+type MexcTrade = {
+  price: string
+  quantity: string
+  tradeType: MexcTradeType
+  time: string | number
+  tradeId?: string
+}
+
+enum MexcTradeType {
+  Buy = 1,
+  Sell = 2
+}
+
+export type MexcDepthSnapshotMessage = {
+  channel: MexcChannelWithSymbol<'spot@public.aggre.depth.snapshot.v3.api.pb@10ms'>
+  symbol: string
+  generated: true
+  publicAggreDepthsSnapshot: MexcAggreDepthsSnapshot
+}
+
+type MexcAggreDepthsSnapshot = {
+  asks: MexcDepthSnapshotLevel[]
+  bids: MexcDepthSnapshotLevel[]
+  lastUpdateId: number
+  timestamp: number
+}
+
+type MexcDepthSnapshotLevel = [string, string]
+
+type MexcDepthUpdateMessage = MexcProtobufMessage<'spot@public.aggre.depth.v3.api.pb@10ms'> & {
+  generated?: undefined
+  publicAggreDepths: MexcAggreDepths
+}
+
+type MexcAggreDepths = {
+  eventType: 'spot@public.aggre.depth.v3.api.pb@10ms'
+  asks?: MexcPriceLevel[]
+  bids?: MexcPriceLevel[]
+  fromVersion: string
+  toVersion: string
+  lastOrderCreateTime?: string | number
+}
+
+type MexcPriceLevel = {
+  price: string
+  quantity: string
+}
+
+type MexcBookTickerMessage = MexcProtobufMessage<'spot@public.aggre.bookTicker.v3.api.pb@10ms'> & {
+  publicAggreBookTicker: {
+    bidPrice?: string
+    bidQuantity?: string
+    askPrice?: string
+    askQuantity?: string
+    version?: string
+    lastOrderCreateTime?: string | number
+  }
+}
+
+type MexcDepthInfo = {
+  isContinuityValidated?: boolean
+  currentBookVersion?: number
+  snapshotEmitted?: boolean
+  updates: CircularBuffer<MexcDepthUpdateMessage>
+}