tardis-machine 18.4.2 → 18.6.0

package/node_modules/tardis-dev/src/realtimefeeds/coinbasederivatives.ts

@@ -1,5 +1,5 @@
 import { Writable } from 'node:stream'
-import { batch, getJSON, optimizeFilters } from '../handy.ts'
+import { getJSON, optimizeFilters } from '../handy.ts'
 import { Filter } from '../types.ts'
 import { MultiConnectionRealTimeFeedBase, PoolingClientBase, RealTimeFeedBase } from './realtimefeed.ts'
 
@@ -12,16 +12,8 @@ export class CoinbaseDerivativesRealTimeFeed extends MultiConnectionRealTimeFeed
       }
     }
 
-    const symbols = [...new Set(wsFilters.flatMap((f) => f.symbols ?? []))]
-    const groups: Iterable<string[]> = symbols.length > 0 ? batch(symbols, 25) : wsFilters.length > 0 ? [[]] : []
-    for (const group of groups) {
-      const groupedFilters = wsFilters
-        .map((f) => ({
-          ...f,
-          symbols: f.symbols?.filter((s) => group.includes(s))
-        }))
-        .filter((f) => f.symbols?.length || ['heartbeats', 'subscriptions'].includes(f.channel))
-      yield new CoinbaseDerivativesSingleConnectionRealTimeFeed(exchange, groupedFilters, timeoutIntervalMS, onError)
+    if (wsFilters.length > 0) {
+      yield new CoinbaseDerivativesSingleConnectionRealTimeFeed(exchange, wsFilters, timeoutIntervalMS, onError)
     }
 
     const productFilters = filters.filter((f) => f.channel === 'products')

package/node_modules/tardis-dev/src/realtimefeeds/gateiofutures.ts

@@ -9,16 +9,66 @@ export class GateIOFuturesRealTimeFeed extends MultiConnectionRealTimeFeedBase {
     )
 
     const inverseContractsFilters = filters.reduce(
-      this._only((s) => s.endsWith('_USDT') === false),
+      this._only((s) => s.endsWith('_USD')),
       [] as Filter<string>[]
     )
 
     if (linearContractsFilters.length > 0) {
-      yield new GateIOFuturesSingleConnectionRealTimeFeed('usdt', exchange, linearContractsFilters, timeoutIntervalMS, onError)
+      yield new GateIOFuturesSingleConnectionRealTimeFeed(
+        'wss://fx-ws.gateio.ws/v4/ws/usdt',
+        400,
+        exchange,
+        linearContractsFilters,
+        timeoutIntervalMS,
+        onError
+      )
     }
 
     if (inverseContractsFilters.length > 0) {
-      yield new GateIOFuturesSingleConnectionRealTimeFeed('btc', exchange, inverseContractsFilters, timeoutIntervalMS, onError)
+      yield new GateIOFuturesSingleConnectionRealTimeFeed(
+        'wss://fx-ws.gateio.ws/v4/ws/btc',
+        400,
+        exchange,
+        inverseContractsFilters,
+        timeoutIntervalMS,
+        onError
+      )
+    }
+
+    const usd1ContractsFilters = filters.reduce(
+      this._only((s) => s.endsWith('_USD1')),
+      [] as Filter<string>[]
+    )
+    if (usd1ContractsFilters.length > 0) {
+      yield new GateIOFuturesSingleConnectionRealTimeFeed(
+        'wss://fx-ws.gateio.ws/v4/ws/usd1',
+        400,
+        exchange,
+        usd1ContractsFilters,
+        timeoutIntervalMS,
+        onError
+      )
+    }
+
+    // Gate has no public liquidations channel for delivery futures
+    if (filters.every((filter) => filter.channel === 'public_liquidates' && filter.symbols!.every((s) => /_USDT_\d{8}$/.test(s)))) {
+      throw new Error('Gate.io delivery futures do not support public_liquidates')
+    }
+    const deliveryContractsFilters = filters
+      .filter((filter) => filter.channel !== 'public_liquidates')
+      .reduce(
+        this._only((s) => /_USDT_\d{8}$/.test(s)),
+        [] as Filter<string>[]
+      )
+    if (deliveryContractsFilters.length > 0) {
+      yield new GateIOFuturesSingleConnectionRealTimeFeed(
+        'wss://fx-ws.gateio.ws/v4/ws/delivery/usdt',
+        100,
+        exchange,
+        deliveryContractsFilters,
+        timeoutIntervalMS,
+        onError
+      )
     }
   }
 
@@ -27,6 +77,11 @@ export class GateIOFuturesRealTimeFeed extends MultiConnectionRealTimeFeedBase {
       if (!current.symbols || current.symbols.length === 0) {
         throw new Error('GateIOFuturesRealTimeFeed requires explicitly specified symbols when subscribing to live feed')
       }
+      for (const symbol of current.symbols) {
+        if (!/_(USD|USDT|USD1)$/.test(symbol) && !/_USDT_\d{8}$/.test(symbol)) {
+          throw new Error(`Unsupported Gate.io futures symbol: ${symbol}`)
+        }
+      }
 
       const symbols = current.symbols.filter(filter)
       if (symbols.length > 0) {
@@ -41,28 +96,29 @@ export class GateIOFuturesRealTimeFeed extends MultiConnectionRealTimeFeedBase {
 }
 
 class GateIOFuturesSingleConnectionRealTimeFeed extends RealTimeFeedBase {
-  protected readonly wssURL: string
   protected readonly extraHeaders = { 'X-Gate-Size-Decimal': '1' }
+  private readonly bookSequences = new Map<string, number>()
 
   constructor(
-    wsURLSuffix: string,
+    protected readonly wssURL: string,
+    private readonly orderBookDepth: 100 | 400,
     exchange: string,
     filters: Filter<string>[],
     timeoutIntervalMS?: number,
     onError?: (error: Error) => void
   ) {
     super(exchange, filters, timeoutIntervalMS, onError)
-    this.wssURL = `wss://fx-ws.gateio.ws/v4/ws/${wsURLSuffix}`
   }
 
   protected mapToSubscribeMessages(filters: Filter<string>[]): any[] {
+    this.bookSequences.clear()
     const payload = filters.flatMap((filter) => {
-      if (filter.channel === 'order_book') {
+      if (filter.channel === 'order_book' || filter.channel === 'obu') {
         return filter.symbols!.map((symbol) => {
           return {
             event: 'subscribe',
             channel: `futures.${filter.channel}`,
-            payload: [symbol, '20', '0'],
+            payload: filter.channel === 'obu' ? [`ob.${symbol}.${this.orderBookDepth}`] : [symbol, '20', '0'],
             time: Math.floor(new Date().valueOf() / 1000)
           }
         })
@@ -81,6 +137,22 @@ class GateIOFuturesSingleConnectionRealTimeFeed extends RealTimeFeedBase {
     return payload
   }
 
+  protected onMessage(message: any) {
+    if (message.channel !== 'futures.obu' || (message.event !== 'update' && message.event !== 'all')) {
+      return
+    }
+
+    const { result } = message
+    const previous = this.bookSequences.get(result.s)
+    if (
+      !Number.isSafeInteger(result.u) ||
+      (result.full !== true && (previous === undefined || result.U !== previous + 1 || result.u < result.U))
+    ) {
+      throw new Error(`Gate.io OBU sequence gap for ${result.s}: previous=${previous}, U=${result.U}, u=${result.u}`)
+    }
+    this.bookSequences.set(result.s, result.u)
+  }
+
   protected messageIsError(message: any): boolean {
     if (message.error !== null && message.error !== undefined) {
       return true

package/node_modules/tardis-dev/src/realtimefeeds/realtimefeed.ts

@@ -5,7 +5,7 @@ import { setTimeout as sleep } from 'node:timers/promises'
 import { createDebug, type DebugLogger } from '../debug.ts'
 import { getProxyAgent, HttpClientError, ONE_SEC_IN_MS, optimizeFilters } from '../handy.ts'
 import { getMessageParser } from '../parsemessage.ts'
-import { createManagedRealTimeIterator, mergeRealTime, type ManagedRealTimeIterator } from '../realtimeiterator.ts'
+import { closeIterator, createManagedRealTimeIterator, mergeRealTime, type ManagedRealTimeIterator } from '../realtimeiterator.ts'
 import { Exchange, Filter } from '../types.ts'
 
 export type RealTimeFeed = {
@@ -19,7 +19,7 @@ export type RealTimeFeed = {
 
 let connectionCounter = 1
 
-export type RealTimeFeedIterable = AsyncIterable<any>
+export type RealTimeFeedIterable = { [Symbol.asyncIterator](): ManagedRealTimeIterator<any> }
 
 type RealTimeFeedConnection = {
   id: number
@@ -95,6 +95,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
       let staleConnectionTimerId: NodeJS.Timeout | undefined
       let pingTimerId: NodeJS.Timeout | undefined
       let connectionError: Error | undefined
+      let delivered = false
 
       try {
         this.manualSnapshotsBuffer.length = 0
@@ -173,6 +174,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
           this.onMessage(messageDeserialized)
 
           yield messageDeserialized
+          delivered = true
 
           if (retries > 0) {
             // reset retries counter as we've received correct message from the connection
@@ -216,33 +218,22 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
       if (connectionError !== undefined) {
         this._onError?.(connectionError)
         retries++
-
-        const MAX_DELAY = 32 * 1000
-        const httpError = connectionError instanceof HttpClientError ? connectionError : undefined
-        const isRateLimited = httpError?.status === 418 || httpError?.status === 429 || connectionError.message.includes('429')
-
-        if (isRateLimited) {
-          const fallbackDelay = httpError?.status === 418 ? 2 * 60 * 1000 : (MAX_DELAY / 2) * retries
-          retryDelay = httpError?.retryAfterMS ?? fallbackDelay
-        } else {
-          retryDelay = Math.min(Math.pow(2, retries - 1) * 1000, MAX_DELAY)
-        }
+        retryDelay = getRetryDelay(connectionError, retries)
 
         this.debug(
-          '(connection id: %d) %s real-time feed connection error, retries count: %d, next retry delay: %dms, rate limited: %s error message: %o',
+          '(connection id: %d) %s real-time feed connection error, retries count: %d, next retry delay: %dms, error message: %o',
           connectionId,
           this._exchange,
           retries,
           retryDelay,
-          isRateLimited,
           connectionError
         )
       }
 
-      yield { __disconnect__: true }
+      yield { __disconnect__: true, error: connectionError, delivered }
 
       if (retryDelay !== undefined) {
-        await this._wait(retryDelay, this._closeController.signal)
+        await wait(retryDelay, this._closeController.signal)
       }
     }
   }
@@ -343,7 +334,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
 
         this._send(connection, message)
         if (this.throttleSubscribeMS > 0) {
-          await this._wait(this.throttleSubscribeMS, connection.controller.signal)
+          await wait(this.throttleSubscribeMS, connection.controller.signal)
         }
       }
 
@@ -352,7 +343,7 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
       const messagesBeforeSnapshots = subscribeMessages.length === 0 ? 0 : subscribeMessages.length + 1
       // Wait for subscription acknowledgements and the first data message before fetching snapshots.
       while (shouldCancel() === false && connection.receivedMessagesCount < messagesBeforeSnapshots) {
-        await this._wait(100, connection.controller.signal)
+        await wait(100, connection.controller.signal)
       }
 
       if (shouldCancel()) {
@@ -411,18 +402,31 @@ export abstract class RealTimeFeedBase implements RealTimeFeedIterable {
       this.manualSnapshotsBuffer.length = 0
     }
   }
+}
 
-  private async _wait(delayMS: number, signal: AbortSignal) {
-    try {
-      await sleep(delayMS, undefined, { signal })
-    } catch (error) {
-      if (signal.aborted === false) {
-        throw error
-      }
+async function wait(delayMS: number, signal: AbortSignal) {
+  try {
+    await sleep(delayMS, undefined, { signal })
+  } catch (error) {
+    if (signal.aborted === false) {
+      throw error
     }
   }
 }
 
+function getRetryDelay(error: Error, retries: number) {
+  const MAX_DELAY = 32 * 1000
+  const httpError = error instanceof HttpClientError ? error : undefined
+  const isRateLimited = httpError?.status === 418 || httpError?.status === 429 || error.message.includes('429')
+
+  if (isRateLimited) {
+    const fallbackDelay = httpError?.status === 418 ? 2 * 60 * 1000 : (MAX_DELAY / 2) * retries
+    return httpError?.retryAfterMS ?? fallbackDelay
+  }
+
+  return Math.min(Math.pow(2, retries - 1) * 1000, MAX_DELAY)
+}
+
 function resolveOrAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T | typeof ABORTED> {
   if (signal.aborted) {
     return Promise.resolve(ABORTED)
@@ -449,7 +453,12 @@ function resolveOrAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T
   })
 }
 
+// A disconnect resets all normalized state, so when any connection drops, the feed closes all its connections and pollers,
+// yields one disconnect and opens them again after the retry delay of the failed connection.
 export abstract class MultiConnectionRealTimeFeedBase implements RealTimeFeedIterable {
+  private readonly _closeController = new AbortController()
+  private _connections?: ManagedRealTimeIterator<any>
+
   constructor(
     private readonly _exchange: string,
     private readonly _filters: Filter<string>[],
@@ -458,8 +467,44 @@ export abstract class MultiConnectionRealTimeFeedBase implements RealTimeFeedIte
   ) {}
 
   [Symbol.asyncIterator](): ManagedRealTimeIterator<any> {
-    const realTimeFeeds = Array.from(this._getRealTimeFeeds(this._exchange, this._filters, this._timeoutIntervalMS, this._onError))
-    return mergeRealTime(realTimeFeeds)
+    return createManagedRealTimeIterator(this._stream(), () => {
+      this._closeController.abort()
+      return this._connections?.return()
+    })
+  }
+
+  private async *_stream() {
+    let retries = 0
+
+    while (this._closeController.signal.aborted === false) {
+      this._connections = mergeRealTime(
+        Array.from(this._getRealTimeFeeds(this._exchange, this._filters, this._timeoutIntervalMS, this._onError), untilDisconnect)
+      )
+      let disconnect: { error?: Error; delivered?: boolean } | undefined
+
+      for await (const message of this._connections) {
+        if (message.__disconnect__ === true) {
+          disconnect = message
+          break
+        }
+        yield message
+      }
+
+      if (disconnect === undefined) {
+        return
+      }
+
+      yield { __disconnect__: true }
+
+      if (disconnect.delivered === true) {
+        retries = 0
+      }
+
+      if (disconnect.error !== undefined) {
+        retries++
+        await wait(getRetryDelay(disconnect.error, retries), this._closeController.signal)
+      }
+    }
   }
 
   protected abstract _getRealTimeFeeds(
@@ -470,6 +515,22 @@ export abstract class MultiConnectionRealTimeFeedBase implements RealTimeFeedIte
   ): IterableIterator<RealTimeFeedIterable>
 }
 
+// A connection stops at its disconnect, so it cannot reconnect on its own before the feed restarts all connections.
+function untilDisconnect(feed: RealTimeFeedIterable) {
+  const messages = feed[Symbol.asyncIterator]()
+
+  async function* stream() {
+    for await (const message of messages) {
+      yield message
+      if (message.__disconnect__ === true) {
+        return
+      }
+    }
+  }
+
+  return createManagedRealTimeIterator(stream(), () => closeIterator(messages))
+}
+
 export abstract class PoolingClientBase implements RealTimeFeedIterable {
   protected readonly debug: DebugLogger
   private _tid: NodeJS.Timeout | undefined = undefined

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tardis-machine",
-  "version": "18.4.2",
+  "version": "18.6.0",
   "engines": {
     "node": ">=24.19"
   },
@@ -76,7 +76,7 @@
   "license": "MPL-2.0",
   "dependencies": {
     "debug": "^4.4.3",
-    "tardis-dev": "^18.4.2",
+    "tardis-dev": "^18.6.0",
     "ws": "^8.21.3"
   },
   "bundleDependencies": true,

package/src/ws/subscriptionsmappers.ts

@@ -439,9 +439,20 @@ const gateIOFuturesMapper: SubscriptionMapper = {
   },
 
   map: (message: any) => {
+    const channel = message.channel.split('.')[1]
+    if (channel === 'obu') {
+      return [{ channel, symbols: message.payload.map((stream: string) => stream.slice(3, stream.lastIndexOf('.'))) }]
+    }
+    if (channel === 'order_book') {
+      return [{ channel, symbols: [message.payload[0]] }]
+    }
+    if (channel === 'public_liquidates' && message.payload.includes('!all')) {
+      return [{ channel, symbols: [] }]
+    }
+
     return [
       {
-        channel: message.channel.split('.')[1],
+        channel,
         symbols: message.payload.map((s: any) => {
           if (typeof s === 'string') {
             return s