tardis-dev 16.6.2 → 17.0.0

package/src/mappers/phemex.ts

@@ -41,6 +41,9 @@ function isExcludedFromNormalizedOutput(symbol: string) {
   return symbol === 'sOLUSDT'
 }
 
+// Uppercase normalized spot symbols can be indistinguishable from native Phemex perpetual symbols.
+// Keep this list aligned with tardis-api and tardis-exporter so aliases such as SBTCUSDT can be
+// converted back to sBTCUSDT without changing native perpetual symbols such as SPKUSDT.
 const COINS_STARTING_WITH_S = [
   'SOLUSD',
   'SUSHIUSD',
@@ -122,13 +125,46 @@ const COINS_STARTING_WITH_S = [
   'STXXUSDT',
   'SPYXUSDT',
   'SP500USDT',
-  'SNDKUSDT'
+  'SNDKUSDT',
+  'SKHYUSDT',
+  'SMCIUSDT',
+  'SONYUSDT',
+  'SQQQUSDT',
+  'STRCUSDT'
 ]
+
+// Phemex used the V2 `_p` channels for this short-lived, now delisted PerpetualPilot market family.
+// Exact casing matters: SIRENPUSD was a pilot, while sIRENPUSD was a separate spot instrument.
+const PERPETUAL_PILOT_SYMBOLS = new Set([
+  'ANGLERFISHPUSD',
+  'BNBCARDPUSD',
+  'BNBXBTPUSD',
+  'BUTTCOINPUSD',
+  'CRYPTOAIPUSD',
+  'CTDPUSD',
+  'DOGEAIPUSD',
+  'FULLSENDPUSD',
+  'GHIBLIPUSD',
+  'GREED3PUSD',
+  'IMGPUSD',
+  'MCPOSPUSD',
+  'PAINPUSD',
+  'PERRYPUSD',
+  'SIRENPUSD',
+  'TITCOINPUSD',
+  'TOLYPUSD',
+  'WILDNOUTPUSD'
+])
+
 function getInstrumentType(symbol: string) {
   if (/\d+$/.test(symbol)) {
     return 'future'
   }
 
+  if (symbol.startsWith('s')) {
+    return 'spot'
+  }
+
   if (
     COINS_STARTING_WITH_S.some((coin) => (coin === 'SUSDT' ? symbol === coin : symbol.startsWith(coin))) ||
     symbol.startsWith('S') === false
@@ -140,6 +176,16 @@ function getInstrumentType(symbol: string) {
 }
 
 function getApiSymbolId(symbolId: string) {
+  // These are already exchange-native mixed-case identifiers.
+  if (symbolId.startsWith('s') || symbolId.startsWith('u100') || symbolId.startsWith('c')) {
+    return symbolId
+  }
+
+  // SIRENPUSD must not be mistaken for the uppercase alias of spot sIRENPUSD.
+  if (PERPETUAL_PILOT_SYMBOLS.has(symbolId)) {
+    return symbolId
+  }
+
   const type = getInstrumentType(symbolId)
   if (type === 'spot' && symbolId.startsWith('S')) {
     return symbolId.charAt(0).toLowerCase() + symbolId.slice(1)
@@ -155,13 +201,26 @@ function getApiSymbolId(symbolId: string) {
   return symbolId
 }
 
-function getSymbols(symbols: string[]) {
-  const perpV2Symbols = symbols.filter((s) => getInstrumentType(s) === 'perpetual' && s.endsWith('USDT')).map(getApiSymbolId)
-  const otherSymbols = symbols.filter((s) => getInstrumentType(s) !== 'perpetual' || s.endsWith('USDT') == false).map(getApiSymbolId)
+function splitSymbolsByChannelFamily(symbols: string[]) {
+  const v2Symbols: string[] = []
+  const legacySymbols: string[] = []
+
+  for (const symbol of symbols) {
+    const apiSymbolId = getApiSymbolId(symbol)
+    const usesV2Channels =
+      PERPETUAL_PILOT_SYMBOLS.has(apiSymbolId) ||
+      (getInstrumentType(symbol) === 'perpetual' && (apiSymbolId.endsWith('USDT') || apiSymbolId.endsWith('USDC')))
+
+    if (usesV2Channels) {
+      v2Symbols.push(apiSymbolId)
+    } else {
+      legacySymbols.push(apiSymbolId)
+    }
+  }
 
   return {
-    perpV2Symbols,
-    otherSymbols
+    v2Symbols,
+    legacySymbols
   }
 }
 
@@ -182,20 +241,20 @@ const phemexTradesMapper: Mapper<'phemex', Trade> = {
       ]
     }
 
-    const { perpV2Symbols, otherSymbols } = getSymbols(symbols)
+    const { v2Symbols, legacySymbols } = splitSymbolsByChannelFamily(symbols)
 
     const filters = []
 
-    if (perpV2Symbols.length > 0) {
+    if (v2Symbols.length > 0) {
       filters.push({
         channel: 'trades_p',
-        symbols: perpV2Symbols
+        symbols: v2Symbols
       } as const)
     }
-    if (otherSymbols.length > 0) {
+    if (legacySymbols.length > 0) {
       filters.push({
         channel: 'trades',
-        symbols: otherSymbols
+        symbols: legacySymbols
       } as const)
     }
 
@@ -273,19 +332,19 @@ const phemexBookChangeMapper: Mapper<'phemex', BookChange> = {
       ]
     }
 
-    const { perpV2Symbols, otherSymbols } = getSymbols(symbols)
+    const { v2Symbols, legacySymbols } = splitSymbolsByChannelFamily(symbols)
     const filters = []
 
-    if (perpV2Symbols.length > 0) {
+    if (v2Symbols.length > 0) {
       filters.push({
         channel: 'orderbook_p',
-        symbols: perpV2Symbols
+        symbols: v2Symbols
       } as const)
     }
-    if (otherSymbols.length > 0) {
+    if (legacySymbols.length > 0) {
       filters.push({
         channel: 'book',
-        symbols: otherSymbols
+        symbols: legacySymbols
       } as const)
     }
 
@@ -345,19 +404,19 @@ class PhemexDerivativeTickerMapper implements Mapper<'phemex', DerivativeTicker>
       ]
     }
 
-    const { perpV2Symbols, otherSymbols } = getSymbols(symbols)
+    const { v2Symbols, legacySymbols } = splitSymbolsByChannelFamily(symbols)
     const filters = []
 
-    if (perpV2Symbols.length > 0) {
+    if (v2Symbols.length > 0) {
       filters.push({
         channel: 'perp_market24h_pack_p',
-        symbols: perpV2Symbols
+        symbols: v2Symbols
       } as const)
     }
-    if (otherSymbols.length > 0) {
+    if (legacySymbols.length > 0) {
       filters.push({
         channel: 'market24h',
-        symbols: otherSymbols
+        symbols: legacySymbols
       } as const)
     }
 

package/src/replay.ts

@@ -1,14 +1,14 @@
 import { createReadStream } from 'node:fs'
 import { rm } from 'node:fs/promises'
-import { EventEmitter } from 'events'
+import { EventEmitter, once } from 'events'
 import { Worker } from 'worker_threads'
 import { constants, createGunzip, createZstdDecompress } from 'zlib'
-import { BinarySplitStream } from './binarysplit.ts'
+import { BinarySplitBatchStream } from './binarysplit.ts'
 import { clearCacheSync } from './clearcache.ts'
 import { EXCHANGES, EXCHANGE_CHANNELS_INFO } from './consts.ts'
 import { debug } from './debug.ts'
-import { addDays, createNormalizedSymbolFilter, getFilters, normalizeMessages, parseAsUTCDate, wait } from './handy.ts'
-import { MapperFactory, normalizeBookChanges } from './mappers/index.ts'
+import { addDays, createNormalizedSymbolFilter, getFilters, parseAsUTCDate, wait } from './handy.ts'
+import { Mapper, MapperFactory, normalizeBookChanges } from './mappers/index.ts'
 import { getOptions } from './options.ts'
 import { Disconnect, Exchange, FilterForExchange } from './types.ts'
 import { WorkerJobPayload, WorkerMessage, WorkerSignal } from './worker.ts'
@@ -18,6 +18,19 @@ type ReplayNormalizedMessage<U extends readonly MapperFactory<any, any>[], Z ext
   ? MapperOutput<U[number]> | Disconnect
   : MapperOutput<U[number]>
 
+const DATE_MESSAGE_SPLIT_INDEX = 28
+const CHUNK_SIZE = 256 * 1024
+// Keep decompression lenient for partial gzip responses.
+// See https://github.com/request/request/pull/2492 and https://github.com/node-fetch/node-fetch/pull/239.
+const GZIP_OPTIONS = {
+  chunkSize: CHUNK_SIZE,
+  flush: constants.Z_SYNC_FLUSH,
+  finishFlush: constants.Z_SYNC_FLUSH
+}
+
+type DecodedReplayMessage = { localTimestamp: Date; message: any }
+type ReplayMessage = DecodedReplayMessage | { localTimestamp: Buffer; message: Buffer } | undefined
+
 export async function* replay<T extends Exchange, U extends boolean = false, Z extends boolean = false>({
   exchange,
   from,
@@ -38,6 +51,58 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
       ? { localTimestamp: Buffer; message: Buffer }
       : { localTimestamp: Date; message: any }
 > {
+  let lastMessageWasUndefined = false
+
+  const lineBatches = replayLineBatches({ exchange, from, to, filters, apiKey, autoCleanup, waitWhenDataNotYetAvailable })
+  for await (const bufferLines of lineBatches) {
+    // Decode one line batch in a tight loop, then preserve the public one-message-at-a-time iterator.
+    const messages: ReplayMessage[] = []
+
+    for (let i = 0; i < bufferLines.length; i++) {
+      const bufferLine = bufferLines[i]
+      if (bufferLine.length > 0) {
+        lastMessageWasUndefined = false
+        if (skipDecoding === true) {
+          messages.push({
+            localTimestamp: bufferLine.slice(0, DATE_MESSAGE_SPLIT_INDEX),
+            message: bufferLine.slice(DATE_MESSAGE_SPLIT_INDEX + 1)
+          })
+        } else {
+          const message = parseReplayMessage(exchange, bufferLine)
+          const localTimestamp = parseReplayTimestamp(bufferLine)
+          if (withMicroseconds) {
+            localTimestamp.μs = parseReplayMicroseconds(bufferLine)
+          }
+
+          messages.push({ localTimestamp, message })
+        }
+      } else if (withDisconnects === true && lastMessageWasUndefined === false) {
+        lastMessageWasUndefined = true
+        messages.push(undefined)
+      }
+    }
+
+    // Drain a batch already received from the stream; its iterator surfaces later stream errors on the next read.
+    for (let i = 0; i < messages.length; i++) {
+      yield messages[i] as any
+    }
+  }
+}
+
+type ReplayLineOptions<T extends Exchange> = Pick<
+  ReplayOptions<T, boolean, boolean>,
+  'exchange' | 'from' | 'to' | 'filters' | 'apiKey' | 'autoCleanup' | 'waitWhenDataNotYetAvailable'
+>
+
+async function* replayLineBatches<T extends Exchange>({
+  exchange,
+  from,
+  to,
+  filters,
+  apiKey = undefined,
+  autoCleanup = undefined,
+  waitWhenDataNotYetAvailable = undefined
+}: ReplayLineOptions<T>): AsyncIterableIterator<Buffer[]> {
   validateReplayOptions(exchange, from, to, filters)
 
   const fromDate = parseAsUTCDate(from)
@@ -78,23 +143,6 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
   })
 
   try {
-    // date is always formatted to have length of 28 so we can skip looking for first space in line and use it
-    // as hardcoded value
-    const DATE_MESSAGE_SPLIT_INDEX = 28
-
-    // more lenient gzip decompression
-    // see https://github.com/request/request/pull/2492 and https://github.com/node-fetch/node-fetch/pull/239
-
-    const CHUNK_SIZE = 256 * 1024
-    const GZIP_OPTIONS = {
-      chunkSize: CHUNK_SIZE,
-      flush: constants.Z_SYNC_FLUSH,
-      finishFlush: constants.Z_SYNC_FLUSH
-    }
-
-    // helper flag that helps us not yielding two subsequent undefined/disconnect messages
-    let lastMessageWasUndefined = false
-
     let currentSliceDate = new Date(fromDate)
     // iterate over every minute in <=from,to> date range
     // get cached slice paths, read them as file streams, decompress, split by new lines and yield as messages
@@ -113,9 +161,9 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
         }
 
         if (cachedSlice === undefined) {
-          // if response for requested date is not ready yet wait 100ms and try again
+          // if the requested slice is not ready yet, wait for the worker to report another cached slice
           debug('waiting for slice: %s, exchange: %s', sliceKey, exchange)
-          await wait(100)
+          await once(worker, 'message')
         }
       }
 
@@ -130,7 +178,7 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
           linesStream.destroy(err)
         })
         // and split by new line
-        .pipe(new BinarySplitStream())
+        .pipe(new BinarySplitBatchStream())
         .on('error', function onBinarySplitStreamError(err) {
           debug('binary split stream error %o', err)
           linesStream.destroy(err)
@@ -138,47 +186,9 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
 
       let linesCount = 0
 
-      for await (const bufferLine of linesStream as unknown as Iterable<Buffer>) {
-        linesCount++
-        if (bufferLine.length > 0) {
-          lastMessageWasUndefined = false
-          // as any due to https://github.com/Microsoft/TypeScript/issues/24929
-          if (skipDecoding === true) {
-            yield {
-              localTimestamp: bufferLine.slice(0, DATE_MESSAGE_SPLIT_INDEX),
-              message: bufferLine.slice(DATE_MESSAGE_SPLIT_INDEX + 1)
-            } as any
-          } else {
-            let messageString = bufferLine.toString('utf8', DATE_MESSAGE_SPLIT_INDEX + 1)
-
-            // hack to handle huobi long numeric id for trades
-            if (exchange.startsWith('huobi-') && messageString.includes('.trade.detail')) {
-              messageString = messageString.replace(/"id":([0-9]+),/g, '"id":"$1",')
-            }
-            // hack to handle upbit long numeric id for trades
-            if (exchange === 'upbit' && messageString.includes('sequential_id')) {
-              messageString = messageString.replace(/"sequential_id":([0-9]+),/g, '"sequential_id":"$1",')
-            }
-
-            const message = JSON.parse(messageString)
-
-            const localTimestamp = new Date(bufferLine.toString('utf8', 0, DATE_MESSAGE_SPLIT_INDEX))
-            if (withMicroseconds) {
-              localTimestamp.μs = parseReplayMicroseconds(bufferLine)
-            }
-
-            yield {
-              // when skipDecoding is not set, decode timestamp to Date and message to object
-              localTimestamp,
-              message
-            } as any
-          }
-          // ignore empty lines unless withDisconnects is set to true
-          // do not yield subsequent undefined messages
-        } else if (withDisconnects === true && lastMessageWasUndefined === false) {
-          lastMessageWasUndefined = true
-          yield undefined as any
-        }
+      for await (const bufferLines of linesStream as AsyncIterable<Buffer[]>) {
+        linesCount += bufferLines.length
+        yield bufferLines
       }
 
       debug('processed slice: %s, exchange: %s, count: %d', sliceKey, exchange, linesCount)
@@ -232,6 +242,21 @@ export async function* replay<T extends Exchange, U extends boolean = false, Z e
   }
 }
 
+function parseReplayMessage(exchange: Exchange, bufferLine: Buffer) {
+  let messageString = bufferLine.toString('utf8', DATE_MESSAGE_SPLIT_INDEX + 1)
+
+  // hack to handle huobi long numeric id for trades
+  if (exchange.startsWith('huobi-') && messageString.includes('.trade.detail')) {
+    messageString = messageString.replace(/"id":([0-9]+),/g, '"id":"$1",')
+  }
+  // hack to handle upbit long numeric id for trades
+  if (exchange === 'upbit' && messageString.includes('sequential_id')) {
+    messageString = messageString.replace(/"sequential_id":([0-9]+),/g, '"sequential_id":"$1",')
+  }
+
+  return JSON.parse(messageString)
+}
+
 async function cleanupSlice(slicePath: string) {
   try {
     await rm(slicePath, { force: true })
@@ -244,6 +269,29 @@ function parseReplayMicroseconds(bufferLine: Buffer) {
   return (bufferLine[23] - 48) * 100 + (bufferLine[24] - 48) * 10 + (bufferLine[25] - 48)
 }
 
+function parseReplayTimestamp(bufferLine: Buffer) {
+  // Recorder writes yyyy-MM-ddTHH:mm:ss.fffffffZ.
+  return new Date(
+    Date.UTC(
+      parseReplayTwoDigits(bufferLine, 0) * 100 + parseReplayTwoDigits(bufferLine, 2),
+      parseReplayTwoDigits(bufferLine, 5) - 1,
+      parseReplayTwoDigits(bufferLine, 8),
+      parseReplayTwoDigits(bufferLine, 11),
+      parseReplayTwoDigits(bufferLine, 14),
+      parseReplayTwoDigits(bufferLine, 17),
+      parseReplayThreeDigits(bufferLine, 20)
+    )
+  )
+}
+
+function parseReplayTwoDigits(bufferLine: Buffer, start: number) {
+  return (bufferLine[start] - 48) * 10 + (bufferLine[start + 1] - 48)
+}
+
+function parseReplayThreeDigits(bufferLine: Buffer, start: number) {
+  return (bufferLine[start] - 48) * 100 + (bufferLine[start + 1] - 48) * 10 + (bufferLine[start + 2] - 48)
+}
+
 // gracefully terminate worker
 async function terminateWorker(worker: Worker, waitTimeout: number) {
   let cancelWait = () => {}
@@ -279,9 +327,9 @@ export function replayNormalized<T extends Exchange, U extends MapperFactory<T,
 
   validateReplayNormalizedOptions(fromDate, normalizers)
 
-  const createMappers = (localTimestamp: Date) => normalizers.map((m) => m(exchange, localTimestamp))
-  const mappers = createMappers(fromDate)
-  const filters = getFilters(mappers, symbols)
+  const createMappersAt = (localTimestamp: Date) => normalizers.map((normalizer) => normalizer(exchange, localTimestamp))
+  const initialMappers = createMappersAt(fromDate)
+  const filters = getFilters(initialMappers, symbols)
 
   // filter normalized messages by symbol as some exchanges do not provide server side filtering so we could end up with messages
   // for symbols we've not requested for
@@ -289,19 +337,17 @@ export function replayNormalized<T extends Exchange, U extends MapperFactory<T,
 
   if (segments.length <= 1) {
     const filter = createNormalizedSymbolFilter(symbols, filters)
-    const messages = replay({
+    const lineBatches = replayLineBatches({
       exchange,
       from,
       to,
-      withDisconnects: true,
       filters,
       apiKey,
-      withMicroseconds: true,
       autoCleanup,
       waitWhenDataNotYetAvailable
     })
 
-    return normalizeMessages(exchange, undefined, messages, mappers, createMappers, withDisconnectMessages, filter)
+    return normalizeReplayLineBatches(exchange, undefined, lineBatches, initialMappers, createMappersAt, withDisconnectMessages, filter)
   }
 
   return replayNormalizedSegments()
@@ -309,23 +355,116 @@ export function replayNormalized<T extends Exchange, U extends MapperFactory<T,
   async function* replayNormalizedSegments() {
     for (let i = 0; i < segments.length; i++) {
       const { from: segmentFrom, to: segmentTo } = segments[i]
-      const segmentMappers = i === 0 ? mappers : createMappers(segmentFrom)
+      const segmentMappers = i === 0 ? initialMappers : createMappersAt(segmentFrom)
       const segmentFilters = getFilters(segmentMappers, symbols)
       const segmentFilter = createNormalizedSymbolFilter(symbols, segmentFilters)
 
-      const segmentMessages = replay({
+      const segmentLineBatches = replayLineBatches({
         exchange,
         from: segmentFrom.toISOString(),
         to: segmentTo.toISOString(),
-        withDisconnects: true,
         filters: segmentFilters,
         apiKey,
-        withMicroseconds: true,
         autoCleanup,
         waitWhenDataNotYetAvailable
       })
 
-      yield* normalizeMessages(exchange, undefined, segmentMessages, segmentMappers, createMappers, withDisconnectMessages, segmentFilter)
+      yield* normalizeReplayLineBatches(
+        exchange,
+        undefined,
+        segmentLineBatches,
+        segmentMappers,
+        createMappersAt,
+        withDisconnectMessages,
+        segmentFilter
+      )
+    }
+  }
+}
+
+async function* normalizeReplayLineBatches(
+  exchange: Exchange,
+  symbols: string[] | undefined,
+  lineBatches: AsyncIterableIterator<Buffer[]>,
+  initialMappers: Mapper<any, any>[],
+  createMappersAt: (localTimestamp: Date) => Mapper<any, any>[],
+  withDisconnectMessages: boolean | undefined,
+  filter?: (symbol: string) => boolean
+) {
+  // This intentionally keeps mapper calls lazy. Custom normalizers must not process later raw messages before the consumer asks for them.
+  let previousLocalTimestamp: Date | undefined
+  let activeMappers: Mapper<any, any>[] | undefined = initialMappers
+  if (activeMappers.length === 0) {
+    throw new Error(`Can't normalize data without any normalizers provided`)
+  }
+
+  for await (const bufferLines of lineBatches) {
+    const decodedMessages: (DecodedReplayMessage | undefined)[] = []
+    let decodingFailed = false
+    let decodingError: unknown
+    try {
+      for (let i = 0; i < bufferLines.length; i++) {
+        const bufferLine = bufferLines[i]
+        if (bufferLine.length === 0) {
+          decodedMessages.push(undefined)
+        } else {
+          const message = parseReplayMessage(exchange, bufferLine)
+          const localTimestamp = parseReplayTimestamp(bufferLine)
+          localTimestamp.μs = parseReplayMicroseconds(bufferLine)
+          decodedMessages.push({ localTimestamp, message })
+        }
+      }
+    } catch (error) {
+      decodingFailed = true
+      decodingError = error
+    }
+
+    for (let i = 0; i < decodedMessages.length; i++) {
+      const decodedMessage = decodedMessages[i]
+      if (decodedMessage === undefined) {
+        if (activeMappers === undefined) {
+          continue
+        }
+
+        activeMappers = undefined
+        if (withDisconnectMessages === true && previousLocalTimestamp !== undefined) {
+          const disconnect: Disconnect = {
+            type: 'disconnect',
+            exchange,
+            localTimestamp: previousLocalTimestamp,
+            symbols
+          }
+          yield disconnect as any
+        }
+        continue
+      }
+
+      const { localTimestamp, message } = decodedMessage
+
+      if (activeMappers === undefined) {
+        activeMappers = createMappersAt(localTimestamp)
+      }
+      previousLocalTimestamp = localTimestamp
+
+      for (let mapperIndex = 0; mapperIndex < activeMappers.length; mapperIndex++) {
+        const mapper = activeMappers[mapperIndex]
+        if (mapper.canHandle(message)) {
+          const mappedMessages = mapper.map(message, localTimestamp)
+          if (!mappedMessages) {
+            continue
+          }
+
+          for (const normalizedMessage of mappedMessages) {
+            if (filter === undefined || filter(normalizedMessage.symbol)) {
+              yield normalizedMessage
+            }
+          }
+        }
+      }
+    }
+
+    if (decodingFailed) {
+      throw decodingError
     }
   }
 }