tardis-machine 18.2.2 → 18.3.1

package/node_modules/tardis-dev/src/realtimefeeds/bitvavo.ts

@@ -0,0 +1,107 @@
+import crypto from 'crypto'
+import { Filter } from '../types.ts'
+import { RealTimeFeedBase } from './realtimefeed.ts'
+
+export class BitvavoRealTimeFeed extends RealTimeFeedBase {
+  protected readonly wssURL = 'wss://ws-mdpro.bitvavo.com/v2/'
+  protected readonly throttleSubscribeMS = 25
+  private nextRequestId = 1
+  private pendingAuthentication?: {
+    resolve: () => void
+    reject: (error: Error) => void
+  }
+
+  protected mapToSubscribeMessages(filters: Filter<string>[]) {
+    this.getCredentials()
+
+    const channels = filters
+      .filter((filter) => filter.channel !== 'getBook')
+      .map((filter) => {
+        if (!filter.symbols || filter.symbols.length === 0) {
+          throw new Error('BitvavoRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
+        }
+
+        return {
+          name: filter.channel === 'trade' ? 'trades' : filter.channel,
+          markets: filter.symbols
+        }
+      })
+
+    const snapshotRequests = (filters.find(({ channel }) => channel === 'getBook')?.symbols ?? []).map((symbol) => ({
+      action: 'getBook',
+      requestId: this.nextRequestId++,
+      market: symbol,
+      depth: 1000
+    }))
+
+    return [...(channels.length === 0 ? [] : [{ action: 'subscribe', channels }]), ...snapshotRequests]
+  }
+
+  protected async onConnected() {
+    const { key, secret } = this.getCredentials()
+    const timestamp = Date.now()
+    const signature = crypto.createHmac('sha256', secret).update(`${timestamp}GET/v2/websocket`).digest('hex')
+
+    let resolveAuthentication!: () => void
+    let rejectAuthentication!: (error: Error) => void
+    const authenticationResponse = new Promise<void>((resolve, reject) => {
+      resolveAuthentication = resolve
+      rejectAuthentication = reject
+    })
+    const pendingAuthentication = {
+      resolve: resolveAuthentication,
+      reject: rejectAuthentication
+    }
+    this.pendingAuthentication = pendingAuthentication
+
+    const timeout = setTimeout(() => {
+      pendingAuthentication.reject(new Error('Bitvavo Market Data Pro authentication timed out'))
+    }, 10000)
+
+    this.send({ action: 'authenticate', key, signature, timestamp })
+
+    try {
+      await authenticationResponse
+    } finally {
+      clearTimeout(timeout)
+      if (this.pendingAuthentication === pendingAuthentication) {
+        this.pendingAuthentication = undefined
+      }
+    }
+  }
+
+  protected onMessage(message: BitvavoRealtimeMessage) {
+    if (message.event !== 'authenticate') {
+      return
+    }
+
+    if (message.authenticated === true) {
+      this.pendingAuthentication?.resolve()
+    } else {
+      this.pendingAuthentication?.reject(new Error('Bitvavo Market Data Pro authentication failed'))
+    }
+  }
+
+  protected messageIsError(message: BitvavoRealtimeMessage) {
+    const isError = message.event === 'error' || message.error !== undefined
+    if (isError) {
+      this.pendingAuthentication?.reject(new Error(`Bitvavo Market Data Pro error: ${JSON.stringify(message)}`))
+    }
+    return isError
+  }
+
+  private getCredentials() {
+    const key = process.env.BITVAVO_API_KEY
+    const secret = process.env.BITVAVO_API_SECRET
+    if (key === undefined || secret === undefined) {
+      throw new Error('BitvavoRealTimeFeed requires BITVAVO_API_KEY and BITVAVO_API_SECRET environment variables')
+    }
+    return { key, secret }
+  }
+}
+
+type BitvavoRealtimeMessage = {
+  event?: string
+  authenticated?: boolean
+  error?: unknown
+}

package/node_modules/tardis-dev/src/realtimefeeds/index.ts

@@ -10,7 +10,9 @@ import {
 import { BinanceDexRealTimeFeed } from './binancedex.ts'
 import { BitfinexRealTimeFeed } from './bitfinex.ts'
 import { BitflyerRealTimeFeed } from './bitflyer.ts'
+import { BithumbRealTimeFeed } from './bithumb.ts'
 import { BitstampRealTimeFeed } from './bitstamp.ts'
+import { BitvavoRealTimeFeed } from './bitvavo.ts'
 import { BybitOptionsRealTimeDataFeed, BybitRealTimeDataFeed, BybitSpotRealTimeDataFeed } from './bybit.ts'
 import { CoinbaseRealTimeFeed } from './coinbase.ts'
 import { CryptofacilitiesRealTimeFeed } from './cryptofacilities.ts'
@@ -51,7 +53,7 @@ import { DydxV4RealTimeFeed } from './dydx_v4.ts'
 import { BitgetFuturesRealTimeFeed, BitgetRealTimeFeed } from './bitget.ts'
 import { CoinbaseInternationalRealTimeFeed } from './coinbaseinternational.ts'
 import { HyperliquidRealTimeFeed } from './hyperliquid.ts'
-import { LighterRealTimeFeed } from './lighter.ts'
+import { LighterRealTimeFeed, LighterRhRealTimeFeed } from './lighter.ts'
 import { BullishRealTimeFeed } from './bullish.ts'
 import { MexcFuturesRealTimeFeed } from './mexcfutures.ts'
 import { MexcRealTimeFeed } from './mexc.ts'
@@ -73,7 +75,9 @@ const realTimeFeedsMap: {
   bitfinex: BitfinexRealTimeFeed,
   'bitfinex-derivatives': BitfinexRealTimeFeed,
   bitflyer: BitflyerRealTimeFeed,
+  bithumb: BithumbRealTimeFeed,
   bitstamp: BitstampRealTimeFeed,
+  bitvavo: BitvavoRealTimeFeed,
   coinbase: CoinbaseRealTimeFeed,
   cryptofacilities: CryptofacilitiesRealTimeFeed,
   deribit: DeribitRealTimeDataFeed,
@@ -121,6 +125,7 @@ const realTimeFeedsMap: {
   'coinbase-international': CoinbaseInternationalRealTimeFeed,
   hyperliquid: HyperliquidRealTimeFeed,
   lighter: LighterRealTimeFeed,
+  'lighter-rh': LighterRhRealTimeFeed,
   bullish: BullishRealTimeFeed,
   mexc: MexcRealTimeFeed,
   polymarket: PolymarketRealTimeFeed

package/node_modules/tardis-dev/src/realtimefeeds/lighter.ts

@@ -1,9 +1,7 @@
 import { Filter } from '../types.ts'
 import { RealTimeFeedBase } from './realtimefeed.ts'
 
-export class LighterRealTimeFeed extends RealTimeFeedBase {
-  protected wssURL = 'wss://mainnet.zklighter.elliot.ai/stream'
-
+abstract class RealTimeFeed extends RealTimeFeedBase {
   protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
     return filters.flatMap((filter) => {
       if (filter.channel === 'market_stats') {
@@ -15,7 +13,7 @@ export class LighterRealTimeFeed extends RealTimeFeedBase {
       }
 
       if (!filter.symbols || filter.symbols.length === 0) {
-        throw new Error('LighterRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
+        throw new Error(`${this._exchange} RealTimeFeed requires explicitly specified symbols when subscribing to live feed`)
       }
 
       return filter.symbols.map((marketId) => ({
@@ -29,3 +27,11 @@ export class LighterRealTimeFeed extends RealTimeFeedBase {
     return message.error !== undefined
   }
 }
+
+export class LighterRealTimeFeed extends RealTimeFeed {
+  protected wssURL = 'wss://mainnet.zklighter.elliot.ai/stream'
+}
+
+export class LighterRhRealTimeFeed extends RealTimeFeed {
+  protected wssURL = 'wss://api.rh.lighter.xyz/stream'
+}

package/node_modules/tardis-dev/src/realtimefeeds/realtimefeed.ts

@@ -4,6 +4,7 @@ import { PassThrough, Writable } from 'stream'
 import { setTimeout as sleep } from 'node:timers/promises'
 import { createDebug, type DebugLogger } from '../debug.ts'
 import { getProxyAgent, HttpClientError, ONE_SEC_IN_MS, optimizeFilters } from '../handy.ts'
+import { getMessageParser } from '../parsemessage.ts'
 import { createManagedRealTimeIterator, mergeRealTime, type ManagedRealTimeIterator } from '../realtimeiterator.ts'
 import { Exchange, Filter } from '../types.ts'
 
@@ -39,6 +40,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
   protected readonly throttleSubscribeMS: number = 0
   protected readonly manualSnapshotsBuffer: any[] = []
   private readonly _filters: Filter<string>[]
+  private readonly _parseJson: (message: string) => any
   private _wsClientOptions: WebSocket.ClientOptions | ClientRequestArgs
   private _closed = false
   private readonly _closeController = new AbortController()
@@ -53,6 +55,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
     private readonly _onError?: (error: Error) => void
   ) {
     this._filters = optimizeFilters(filters)
+    this._parseJson = getMessageParser(_exchange)
     this.debug = createDebug(`tardis-dev:realtime:${_exchange}`)
 
     this._wsClientOptions = {
@@ -149,11 +152,6 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
             message = this.decompress(message)
           }
 
-          // hack to handle huobi long numeric id for trades
-          if (this._exchange.startsWith('huobi-') && message.includes('.trade.detail')) {
-            message = message.toString().replace(/"id":([0-9]+),/g, '"id":"$1",') as any
-          }
-
           const messageDeserialized = this.parseMessage(message)
 
           if (this.messageIsError(messageDeserialized)) {
@@ -270,7 +268,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
   protected abstract messageIsError(message: any): boolean
 
   protected parseMessage(message: Buffer<ArrayBufferLike>): any {
-    return JSON.parse(message as any)
+    return this._parseJson(message.toString())
   }
 
   protected sendCustomPing: (() => void) | undefined = undefined

package/node_modules/tardis-dev/src/replay.ts

@@ -10,6 +10,7 @@ import { debug } from './debug.ts'
 import { addDays, createNormalizedSymbolFilter, getFilters, parseAsUTCDate, wait } from './handy.ts'
 import { Mapper, MapperFactory, normalizeBookChanges } from './mappers/index.ts'
 import { getOptions } from './options.ts'
+import { getMessageParser } from './parsemessage.ts'
 import { Disconnect, Exchange, FilterForExchange } from './types.ts'
 import { WorkerJobPayload, WorkerMessage, WorkerSignal } from './worker.ts'
 import { createZstdDecompressionStream } from './zstd.ts'
@@ -53,6 +54,7 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
       : { localTimestamp: Date; message: any }
 > {
   let lastMessageWasUndefined = false
+  const parseMessage = getMessageParser(exchange)
 
   const lineBatches = replayLineBatches({ exchange, from, to, filters, apiKey, autoCleanup, waitWhenDataNotYetAvailable })
   for await (const bufferLines of lineBatches) {
@@ -69,7 +71,7 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
             message: bufferLine.slice(DATE_MESSAGE_SPLIT_INDEX + 1)
           })
         } else {
-          const message = parseReplayMessage(exchange, bufferLine)
+          const message = parseMessage(bufferLine.toString('utf8', DATE_MESSAGE_SPLIT_INDEX + 1))
           const localTimestamp = parseReplayTimestamp(bufferLine)
           if (withMicroseconds) {
             localTimestamp.μs = parseReplayMicroseconds(bufferLine)
@@ -244,21 +246,6 @@ async function* replayLineBatches<T extends Exchange>({
   }
 }
 
-function parseReplayMessage(exchange: Exchange, bufferLine: Buffer) {
-  let messageString = bufferLine.toString('utf8', DATE_MESSAGE_SPLIT_INDEX + 1)
-
-  // hack to handle huobi long numeric id for trades
-  if (exchange.startsWith('huobi-') && messageString.includes('.trade.detail')) {
-    messageString = messageString.replace(/"id":([0-9]+),/g, '"id":"$1",')
-  }
-  // hack to handle upbit long numeric id for trades
-  if (exchange === 'upbit' && messageString.includes('sequential_id')) {
-    messageString = messageString.replace(/"sequential_id":([0-9]+),/g, '"sequential_id":"$1",')
-  }
-
-  return JSON.parse(messageString)
-}
-
 async function cleanupSlice(slicePath: string) {
   try {
     await rm(slicePath, { force: true })
@@ -394,6 +381,7 @@ async function* normalizeReplayLineBatches(
   filter?: (symbol: string) => boolean
 ) {
   // This intentionally keeps mapper calls lazy. Custom normalizers must not process later raw messages before the consumer asks for them.
+  const parseMessage = getMessageParser(exchange)
   let previousLocalTimestamp: Date | undefined
   let activeMappers: Mapper<any, any>[] | undefined = initialMappers
   if (activeMappers.length === 0) {
@@ -410,7 +398,7 @@ async function* normalizeReplayLineBatches(
         if (bufferLine.length === 0) {
           decodedMessages.push(undefined)
         } else {
-          const message = parseReplayMessage(exchange, bufferLine)
+          const message = parseMessage(bufferLine.toString('utf8', DATE_MESSAGE_SPLIT_INDEX + 1))
           const localTimestamp = parseReplayTimestamp(bufferLine)
           localTimestamp.μs = parseReplayMicroseconds(bufferLine)
           decodedMessages.push({ localTimestamp, message })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tardis-machine",
-  "version": "18.2.2",
+  "version": "18.3.1",
   "engines": {
     "node": ">=24.19"
   },
@@ -76,7 +76,7 @@
   "license": "MPL-2.0",
   "dependencies": {
     "debug": "^4.4.3",
-    "tardis-dev": "^18.2.3",
+    "tardis-dev": "^18.3.1",
     "ws": "^8.21.3"
   },
   "bundleDependencies": true,

package/src/ws/subscriptionsmappers.ts

@@ -745,6 +745,23 @@ const lighterMapper: SubscriptionMapper = {
   }
 }
 
+const bitvavoMapper: SubscriptionMapper = {
+  canHandle: (message: any) => {
+    return message.action === 'subscribe' || message.action === 'getBook'
+  },
+
+  map: (message: any) => {
+    if (message.action === 'getBook') {
+      return [{ channel: 'getBook', symbols: [message.market] }]
+    }
+
+    return message.channels.map((channel: any) => ({
+      channel: channel.name === 'trades' ? 'trade' : channel.name,
+      symbols: channel.markets
+    }))
+  }
+}
+
 const bullishMapper: SubscriptionMapper = {
   canHandle: (message: any) => {
     return message.method === 'subscribe'
@@ -834,6 +851,8 @@ export const subscriptionsMappers: Record<Exchange, SubscriptionMapper> = {
   'dydx-v4': dydxMapper,
   'huobi-dm-options': huobiMapper,
   upbit: upbitMapper,
+  bithumb: upbitMapper,
+  bitvavo: bitvavoMapper,
   serum: serumMaper,
   'star-atlas': serumMaper,
   mango: serumMaper,
@@ -854,6 +873,7 @@ export const subscriptionsMappers: Record<Exchange, SubscriptionMapper> = {
   'coinbase-international': coinbaseInternationalMapper,
   hyperliquid: hyperliquidMapper,
   lighter: lighterMapper,
+  'lighter-rh': lighterMapper,
   bullish: bullishMapper,
   polymarket: polymarketMapper
 }