tardis-machine 18.4.0 → 18.4.2

package/node_modules/tardis-dev/src/mappers/coinbasederivatives.ts

@@ -0,0 +1,230 @@
+import { asNumberOrUndefined, parseISODateWithMicroseconds, upperCaseSymbols } from '../handy.ts'
+import { BookChange, BookTicker, DerivativeTicker, Trade } from '../types.ts'
+import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
+import { exchangeMappers } from './registry.ts'
+
+export const coinbaseDerivativesMappers = exchangeMappers({
+  'coinbase-derivatives': {
+    trades: () => new CoinbaseDerivativesTradesMapper(),
+    bookChanges: () => new CoinbaseDerivativesBookChangeMapper(),
+    bookTickers: () => new CoinbaseDerivativesBookTickerMapper(),
+    derivativeTickers: () => new CoinbaseDerivativesTickerMapper()
+  }
+})
+
+class CoinbaseDerivativesTradesMapper implements Mapper<'coinbase-derivatives', Trade> {
+  // Besides the subscribe snapshot, Coinbase resends recent-trade snapshots mid-connection, each followed by an update with the preceding older trades.
+  private readonly _snapshotTradeIds = new Map<string, number>()
+
+  canHandle(message: CoinbaseDerivativesTrade) {
+    return message.channel === 'market_trades'
+  }
+
+  getFilters(symbols?: string[]) {
+    return [{ channel: 'market_trades', symbols: upperCaseSymbols(symbols) } as const]
+  }
+
+  *map(message: CoinbaseDerivativesTrade, localTimestamp: Date): IterableIterator<Trade> {
+    for (const event of message.events) {
+      if (event.type === 'snapshot') {
+        for (const trade of event.trades) {
+          const tradeId = Number(trade.trade_id)
+          this._snapshotTradeIds.set(trade.product_id, Math.max(tradeId, this._snapshotTradeIds.get(trade.product_id) ?? tradeId))
+        }
+        continue
+      }
+
+      for (const trade of event.trades) {
+        if (Number(trade.trade_id) <= (this._snapshotTradeIds.get(trade.product_id) ?? 0)) {
+          continue
+        }
+
+        yield {
+          type: 'trade',
+          exchange: 'coinbase-derivatives',
+          symbol: trade.product_id,
+          id: trade.trade_id,
+          price: Number(trade.price),
+          amount: Number(trade.size),
+          side: trade.side === 'BUY' ? 'sell' : 'buy',
+          timestamp: parseISODateWithMicroseconds(trade.time),
+          localTimestamp
+        }
+      }
+    }
+  }
+}
+
+class CoinbaseDerivativesBookChangeMapper implements Mapper<'coinbase-derivatives', BookChange> {
+  canHandle(message: CoinbaseDerivativesBook) {
+    return message.channel === 'l2_data'
+  }
+
+  getFilters(symbols?: string[]) {
+    return [{ channel: 'l2_data', symbols: upperCaseSymbols(symbols) } as const]
+  }
+
+  *map(message: CoinbaseDerivativesBook, localTimestamp: Date): IterableIterator<BookChange> {
+    for (const event of message.events) {
+      yield {
+        type: 'book_change',
+        exchange: 'coinbase-derivatives',
+        symbol: event.product_id,
+        isSnapshot: event.type === 'snapshot',
+        bids: event.updates.filter((u) => u.side === 'bid').map(this.mapBookLevel),
+        asks: event.updates.filter((u) => u.side === 'offer').map(this.mapBookLevel),
+        timestamp: parseISODateWithMicroseconds(
+          event.updates.length > 0 ? event.updates[event.updates.length - 1].event_time : message.timestamp
+        ),
+        localTimestamp
+      }
+    }
+  }
+
+  private mapBookLevel(update: CoinbaseDerivativesBook['events'][number]['updates'][number]) {
+    return {
+      price: Number(update.price_level),
+      amount: Number(update.new_quantity)
+    }
+  }
+}
+
+class CoinbaseDerivativesBookTickerMapper implements Mapper<'coinbase-derivatives', BookTicker> {
+  canHandle(message: CoinbaseDerivativesTicker) {
+    return message.channel === 'ticker'
+  }
+
+  getFilters(symbols?: string[]) {
+    return [{ channel: 'ticker', symbols: upperCaseSymbols(symbols) } as const]
+  }
+
+  *map(message: CoinbaseDerivativesTicker, localTimestamp: Date): IterableIterator<BookTicker> {
+    for (const event of message.events)
+      for (const ticker of event.tickers) {
+        const bidPrice = asNumberOrUndefined(ticker.best_bid)
+        const askPrice = asNumberOrUndefined(ticker.best_ask)
+
+        if (bidPrice === undefined && askPrice === undefined) {
+          continue
+        }
+
+        yield {
+          type: 'book_ticker',
+          exchange: 'coinbase-derivatives',
+          symbol: ticker.product_id,
+          bidPrice,
+          bidAmount: asNumberOrUndefined(ticker.best_bid_quantity),
+          askPrice,
+          askAmount: asNumberOrUndefined(ticker.best_ask_quantity),
+          timestamp: parseISODateWithMicroseconds(message.timestamp),
+          localTimestamp
+        }
+      }
+  }
+}
+
+class CoinbaseDerivativesTickerMapper implements Mapper<'coinbase-derivatives', DerivativeTicker> {
+  private readonly pending = new PendingTickerInfoHelper()
+
+  canHandle(message: CoinbaseDerivativesProducts | CoinbaseDerivativesTicker) {
+    return message.channel === 'products' || message.channel === 'ticker'
+  }
+
+  getFilters(symbols?: string[]) {
+    return [
+      { channel: 'products', symbols: upperCaseSymbols(symbols) } as const,
+      { channel: 'ticker', symbols: upperCaseSymbols(symbols) } as const
+    ]
+  }
+
+  *map(message: CoinbaseDerivativesProducts | CoinbaseDerivativesTicker, localTimestamp: Date) {
+    if (message.channel === 'ticker') {
+      for (const event of message.events)
+        for (const ticker of event.tickers) {
+          const info = this.pending.getPendingTickerInfo(ticker.product_id, 'coinbase-derivatives')
+          info.updateLastPrice(asNumberOrUndefined(ticker.price))
+          info.updateTimestamp(parseISODateWithMicroseconds(message.timestamp))
+          if (info.hasChanged()) {
+            yield info.getSnapshot(localTimestamp)
+          }
+        }
+      return
+    }
+
+    for (const product of message.events) {
+      const info = this.pending.getPendingTickerInfo(product.product_id, 'coinbase-derivatives')
+
+      const details = product.future_product_details
+      info.updateOpenInterest(asNumberOrUndefined(details.open_interest))
+      info.updateFundingRate(asNumberOrUndefined(details.funding_rate))
+      info.updateIndexPrice(asNumberOrUndefined(details.index_price))
+      // funding_time identifies the published rate, not the next funding event. No mark price is supplied.
+      info.updateTimestamp(localTimestamp)
+
+      if (info.hasChanged()) {
+        yield info.getSnapshot(localTimestamp)
+      }
+    }
+  }
+}
+
+type CoinbaseDerivativesMessage<Channel extends string, Event> = {
+  channel: Channel
+  events: Event[]
+}
+
+type CoinbaseDerivativesTrade = CoinbaseDerivativesMessage<
+  'market_trades',
+  {
+    type: CoinbaseDerivativesMessageType
+    trades: {
+      product_id: string
+      trade_id: string
+      price: string
+      size: string
+      side: 'BUY' | 'SELL'
+      time: string
+    }[]
+  }
+>
+type CoinbaseDerivativesBook = CoinbaseDerivativesMessage<
+  'l2_data',
+  {
+    type: CoinbaseDerivativesMessageType
+    product_id: string
+    updates: {
+      side: 'bid' | 'offer'
+      price_level: string
+      new_quantity: string
+      event_time: string
+    }[]
+  }
+> & { timestamp: string }
+
+type CoinbaseDerivativesTicker = CoinbaseDerivativesMessage<
+  'ticker',
+  {
+    tickers: {
+      product_id: string
+      price: string
+      best_bid?: string
+      best_ask?: string
+      best_bid_quantity?: string
+      best_ask_quantity?: string
+    }[]
+  }
+> & { timestamp: string }
+
+type CoinbaseDerivativesProducts = CoinbaseDerivativesMessage<
+  'products',
+  {
+    product_id: string
+    future_product_details: {
+      open_interest: string
+      funding_rate: string
+      index_price: string
+    }
+  }
+> & { generated: true }
+
+type CoinbaseDerivativesMessageType = 'snapshot' | 'update'

package/node_modules/tardis-dev/src/mappers/dydxv4.ts

@@ -111,6 +111,8 @@ class DydxV4BookChangeMapper implements Mapper<'dydx-v4', BookChange> {
 
 class DydxV4DerivativeTickerMapper implements Mapper<'dydx-v4', DerivativeTicker> {
   private readonly pendingTickerInfoHelper = new PendingTickerInfoHelper()
+  // Oracle prices also cover price-only markets such as USDT-USD, so a price waits here until its market appears in a snapshot, trading update or trade.
+  private readonly _unconfirmedOraclePrices = new Map<string, number>()
 
   canHandle(message: DydxMarketsSnapshot | DyDxMarketsUpdate | DyDxTrade) {
     return message.channel === 'v4_markets' || (message.channel === 'v4_trades' && message.type === 'channel_data')
@@ -133,7 +135,7 @@ class DydxV4DerivativeTickerMapper implements Mapper<'dydx-v4', DerivativeTicker
 
   *map(message: DydxMarketsSnapshot | DyDxMarketsUpdate | DyDxTrade, localTimestamp: Date): IterableIterator<DerivativeTicker> {
     if (message.channel === 'v4_trades') {
-      const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(message.id, 'dydx-v4')
+      const pendingTickerInfo = this._getMarketTickerInfo(message.id)
       pendingTickerInfo.updateLastPrice(Number(message.contents.trades[message.contents.trades.length - 1].price))
 
       return
@@ -143,7 +145,7 @@ class DydxV4DerivativeTickerMapper implements Mapper<'dydx-v4', DerivativeTicker
       for (const key in contents) {
         const marketInfo = (contents as any)[key] as DydxMarketsSnapshotContent | DydxMarketTradeUpdate
 
-        const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(key, 'dydx-v4')
+        const pendingTickerInfo = this._getMarketTickerInfo(key)
 
         if (marketInfo.oraclePrice !== undefined) {
           pendingTickerInfo.updateMarkPrice(Number(marketInfo.oraclePrice))
@@ -167,6 +169,12 @@ class DydxV4DerivativeTickerMapper implements Mapper<'dydx-v4', DerivativeTicker
     if (message.type === 'channel_data' && message.contents.oraclePrices !== undefined) {
       for (const key in message.contents.oraclePrices) {
         const oraclePriceInfo = (message.contents.oraclePrices as any)[key] as OraclePriceInfo
+        if (this.pendingTickerInfoHelper.hasPendingTickerInfo(key) === false) {
+          if (oraclePriceInfo.oraclePrice !== undefined) {
+            this._unconfirmedOraclePrices.set(key, Number(oraclePriceInfo.oraclePrice))
+          }
+          continue
+        }
         const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(key, 'dydx-v4')
         if (oraclePriceInfo.oraclePrice !== undefined) {
           pendingTickerInfo.updateMarkPrice(Number(oraclePriceInfo.oraclePrice))
@@ -179,6 +187,16 @@ class DydxV4DerivativeTickerMapper implements Mapper<'dydx-v4', DerivativeTicker
       }
     }
   }
+
+  private _getMarketTickerInfo(market: string) {
+    const pendingTickerInfo = this.pendingTickerInfoHelper.getPendingTickerInfo(market, 'dydx-v4')
+    const oraclePrice = this._unconfirmedOraclePrices.get(market)
+    if (oraclePrice !== undefined) {
+      pendingTickerInfo.updateMarkPrice(oraclePrice)
+      this._unconfirmedOraclePrices.delete(market)
+    }
+    return pendingTickerInfo
+  }
 }
 
 class DydxV4LiquidationsMapper implements Mapper<'dydx-v4', Liquidation> {

package/node_modules/tardis-dev/src/mappers/index.ts

@@ -16,6 +16,7 @@ import { blockchainComMappers } from './blockchaincom.ts'
 import { bullishMappers } from './bullish.ts'
 import { bybitMappers } from './bybit.ts'
 import { coinbaseMappers } from './coinbase.ts'
+import { coinbaseDerivativesMappers } from './coinbasederivatives.ts'
 import { coinbaseInternationalMappers } from './coinbaseinternational.ts'
 import { coinflexMappers } from './coinflex.ts'
 import { cryptoComMappers } from './cryptocom.ts'
@@ -74,6 +75,7 @@ const registeredMappers = mergeExchangeMappers(
   bullishMappers,
   bybitMappers,
   coinbaseMappers,
+  coinbaseDerivativesMappers,
   coinbaseInternationalMappers,
   coinflexMappers,
   cryptoComMappers,

package/node_modules/tardis-dev/src/realtimefeeds/binance.ts

@@ -437,7 +437,13 @@ export class BinanceFuturesRealTimeFeed extends BinanceRealTimeFeedBase {
 
   protected *_getRealTimeFeeds(exchange: string, filters: Filter<string>[], timeoutIntervalMS?: number, onError?: (error: Error) => void) {
     const wsFilters = filters.filter(
-      (f) => f.channel !== 'openInterest' && f.channel !== 'recentTrades' && f.channel !== 'fundingInfo' && f.channel !== 'insuranceBalance'
+      (f) =>
+        f.channel !== 'openInterest' &&
+        f.channel !== 'recentTrades' &&
+        f.channel !== 'fundingInfo' &&
+        f.channel !== 'fundingRate' &&
+        f.channel !== 'insuranceBalance' &&
+        f.channel !== 'tradingSession'
     )
 
     const publicWsFilters = wsFilters.filter((f) => BINANCE_FUTURES_PUBLIC_CHANNELS.has(f.channel))

package/node_modules/tardis-dev/src/realtimefeeds/coinbasederivatives.ts

@@ -0,0 +1,103 @@
+import { Writable } from 'node:stream'
+import { batch, getJSON, optimizeFilters } from '../handy.ts'
+import { Filter } from '../types.ts'
+import { MultiConnectionRealTimeFeedBase, PoolingClientBase, RealTimeFeedBase } from './realtimefeed.ts'
+
+export class CoinbaseDerivativesRealTimeFeed extends MultiConnectionRealTimeFeedBase {
+  protected *_getRealTimeFeeds(exchange: string, filters: Filter<string>[], timeoutIntervalMS?: number, onError?: (error: Error) => void) {
+    const wsFilters = optimizeFilters(filters.filter((f) => f.channel !== 'products'))
+    for (const filter of wsFilters) {
+      if (!['heartbeats', 'subscriptions'].includes(filter.channel) && !filter.symbols?.length) {
+        throw new Error('Coinbase derivatives WebSocket subscriptions require explicit symbols')
+      }
+    }
+
+    const symbols = [...new Set(wsFilters.flatMap((f) => f.symbols ?? []))]
+    const groups: Iterable<string[]> = symbols.length > 0 ? batch(symbols, 25) : wsFilters.length > 0 ? [[]] : []
+    for (const group of groups) {
+      const groupedFilters = wsFilters
+        .map((f) => ({
+          ...f,
+          symbols: f.symbols?.filter((s) => group.includes(s))
+        }))
+        .filter((f) => f.symbols?.length || ['heartbeats', 'subscriptions'].includes(f.channel))
+      yield new CoinbaseDerivativesSingleConnectionRealTimeFeed(exchange, groupedFilters, timeoutIntervalMS, onError)
+    }
+
+    const productFilters = filters.filter((f) => f.channel === 'products')
+    if (productFilters.length > 0) {
+      const allProducts = productFilters.some((f) => !f.symbols?.length)
+      yield new CoinbaseDerivativesProductsClient(
+        exchange,
+        allProducts ? undefined : new Set(productFilters.flatMap((f) => f.symbols!)),
+        onError
+      )
+    }
+  }
+}
+
+class CoinbaseDerivativesSingleConnectionRealTimeFeed extends RealTimeFeedBase {
+  protected wssURL = 'wss://advanced-trade-ws.coinbase.com'
+  private sequence?: number
+
+  protected mapToSubscribeMessages(filters: Filter<string>[]) {
+    return [
+      {
+        type: 'subscribe',
+        channel: 'heartbeats'
+      },
+      ...filters
+        .filter((f) => !['heartbeats', 'subscriptions'].includes(f.channel))
+        .map((f) => ({
+          type: 'subscribe',
+          channel: f.channel === 'l2_data' ? 'level2' : f.channel,
+          product_ids: f.symbols
+        }))
+    ]
+  }
+
+  protected async onConnected() {
+    this.sequence = undefined
+  }
+
+  protected messageIsError(message: { type?: string }) {
+    return message.type === 'error'
+  }
+
+  protected onMessage(message: { sequence_num?: number }) {
+    if (message.sequence_num === undefined) return
+    if (this.sequence !== undefined && message.sequence_num !== this.sequence + 1) {
+      throw new Error(`Coinbase derivatives sequence gap: ${this.sequence} -> ${message.sequence_num}`)
+    }
+    this.sequence = message.sequence_num
+  }
+}
+
+class CoinbaseDerivativesProductsClient extends PoolingClientBase {
+  constructor(
+    exchange: string,
+    private readonly symbols: Set<string> | undefined,
+    onError?: (error: Error) => void
+  ) {
+    super(exchange, 5, onError)
+  }
+
+  protected async poolDataToStream(output: Writable) {
+    for (let offset = 0; !output.destroyed; offset += 100) {
+      const { data } = await getJSON<{ products: Product[]; pagination: { has_next: boolean } }>(
+        `https://api.coinbase.com/api/v3/brokerage/market/products?product_type=FUTURE&limit=100&offset=${offset}`
+      )
+
+      for (const product of data.products) {
+        if (output.destroyed) return
+
+        if (this.symbols === undefined || this.symbols.has(product.product_id)) {
+          output.write({ channel: 'products', generated: true, events: [product] })
+        }
+      }
+      if (!data.pagination.has_next) return
+    }
+  }
+}
+
+type Product = { product_id: string }

package/node_modules/tardis-dev/src/realtimefeeds/index.ts

@@ -15,6 +15,7 @@ import { BitstampRealTimeFeed } from './bitstamp.ts'
 import { BitvavoRealTimeFeed } from './bitvavo.ts'
 import { BybitOptionsRealTimeDataFeed, BybitRealTimeDataFeed, BybitSpotRealTimeDataFeed, BybitSpreadRealTimeDataFeed } from './bybit.ts'
 import { CoinbaseRealTimeFeed } from './coinbase.ts'
+import { CoinbaseDerivativesRealTimeFeed } from './coinbasederivatives.ts'
 import { CryptofacilitiesRealTimeFeed } from './cryptofacilities.ts'
 import { DeribitRealTimeDataFeed } from './deribit.ts'
 import { FtxRealTimeFeed, FtxUSRealTimeFeed } from './ftx.ts'
@@ -78,6 +79,7 @@ const realTimeFeedsMap: {
   bitstamp: BitstampRealTimeFeed,
   bitvavo: BitvavoRealTimeFeed,
   coinbase: CoinbaseRealTimeFeed,
+  'coinbase-derivatives': CoinbaseDerivativesRealTimeFeed,
   cryptofacilities: CryptofacilitiesRealTimeFeed,
   deribit: DeribitRealTimeDataFeed,
   ftx: FtxRealTimeFeed,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tardis-machine",
-  "version": "18.4.0",
+  "version": "18.4.2",
   "engines": {
     "node": ">=24.19"
   },
@@ -76,7 +76,7 @@
   "license": "MPL-2.0",
   "dependencies": {
     "debug": "^4.4.3",
-    "tardis-dev": "^18.4.0",
+    "tardis-dev": "^18.4.2",
     "ws": "^8.21.3"
   },
   "bundleDependencies": true,

package/src/ws/subscriptionsmappers.ts

@@ -58,6 +58,12 @@ const coinbaseMaper: SubscriptionMapper = {
   }
 }
 
+// https://docs.cdp.coinbase.com/coinbase-app/advanced-trade-apis/websocket/websocket-overview
+const coinbaseDerivativesMapper: SubscriptionMapper = {
+  canHandle: (message: any) => message.type === 'subscribe',
+  map: (message: any) => [{ channel: message.channel === 'level2' ? 'l2_data' : message.channel, symbols: message.product_ids }]
+}
+
 // https://docs.deribit.com/v2/#subscription-management
 const deribitMapper: SubscriptionMapper = {
   canHandle: (message: any) => {
@@ -220,6 +226,10 @@ const binanceMapper: SubscriptionMapper = {
       const lastSeparator = param.lastIndexOf('@')
       const firstSeparator = param.indexOf('@')
 
+      if (firstSeparator === -1) {
+        return { channel: param, symbols: undefined }
+      }
+
       return {
         channel: param.slice(firstSeparator + 1, lastSeparator == firstSeparator ? undefined : lastSeparator),
         symbols: [param.slice(0, firstSeparator)]
@@ -811,6 +821,7 @@ const polymarketMapper: SubscriptionMapper = {
 export const subscriptionsMappers: Record<Exchange, SubscriptionMapper> = {
   bitmex: bitmexMapper,
   coinbase: coinbaseMaper,
+  'coinbase-derivatives': coinbaseDerivativesMapper,
   deribit: deribitMapper,
   cryptofacilities: cryptofacilitiesMapper,
   bitstamp: bitstampMapper,