tardis-machine 17.0.0 → 18.0.0

package/node_modules/tardis-dev/src/binarysplit.ts

@@ -0,0 +1,50 @@
+import { Transform } from 'stream'
+import type { TransformCallback } from 'stream'
+
+const NEW_LINE_BYTE = 10
+
+// Inspired by https://github.com/maxogden/binary-split/blob/master/index.js.
+// Batching lines lets replay cross the object-mode async iterator boundary once per decompressed chunk.
+export class BinarySplitBatchStream extends Transform {
+  private _buffered?: Buffer
+
+  constructor() {
+    super({
+      readableObjectMode: true,
+      // A batch can hold a full decompression chunk, so keep read-ahead deliberately small.
+      readableHighWaterMark: 2
+    })
+  }
+
+  _transform(chunk: Buffer, _: string, callback: TransformCallback) {
+    const lines: Buffer[] = []
+    let lineStart = 0
+
+    if (this._buffered !== undefined) {
+      const firstNewLineIndex = chunk.indexOf(NEW_LINE_BYTE)
+
+      if (firstNewLineIndex === -1) {
+        this._buffered = Buffer.concat([this._buffered, chunk])
+        callback()
+        return
+      }
+
+      lines.push(Buffer.concat([this._buffered, chunk.subarray(0, firstNewLineIndex)]))
+      this._buffered = undefined
+      lineStart = firstNewLineIndex + 1
+    }
+
+    let newLineIndex = chunk.indexOf(NEW_LINE_BYTE, lineStart)
+    while (newLineIndex !== -1) {
+      lines.push(chunk.subarray(lineStart, newLineIndex))
+      lineStart = newLineIndex + 1
+      newLineIndex = chunk.indexOf(NEW_LINE_BYTE, lineStart)
+    }
+
+    this._buffered = lineStart < chunk.length ? chunk.subarray(lineStart) : undefined
+    if (lines.length > 0) {
+      this.push(lines)
+    }
+    callback()
+  }
+}

package/node_modules/tardis-dev/src/boundedqueue.ts

@@ -0,0 +1,118 @@
+type PendingRead<T> = {
+  resolve: (result: IteratorResult<T>) => void
+  reject: (error: Error) => void
+}
+
+type PendingWrite<T> = {
+  value: T
+  resolve: (written: boolean) => void
+}
+
+export class BoundedQueue<T> {
+  private readonly values: (T | undefined)[]
+  private readIndex = 0
+  private size = 0
+  private pendingWrites: PendingWrite<T>[] = []
+  private pendingRead?: PendingRead<T>
+  private failure?: Error
+  private open = true
+
+  constructor(private readonly capacity: number) {
+    this.values = new Array(capacity)
+  }
+
+  get isOpen() {
+    return this.open
+  }
+
+  write(value: T): boolean | Promise<boolean> {
+    if (!this.open) {
+      return false
+    }
+
+    if (this.pendingRead !== undefined) {
+      const { resolve } = this.pendingRead
+      this.pendingRead = undefined
+      resolve({ done: false, value })
+      return true
+    }
+
+    if (this.size < this.capacity) {
+      this.values[(this.readIndex + this.size) % this.capacity] = value
+      this.size++
+      return true
+    }
+
+    return new Promise((resolve) => this.pendingWrites.push({ value, resolve }))
+  }
+
+  async next(): Promise<IteratorResult<T>> {
+    if (this.size > 0) {
+      const value = this.values[this.readIndex] as T
+      this.values[this.readIndex] = undefined
+      this.readIndex = (this.readIndex + 1) % this.capacity
+      this.size--
+      const pendingWrite = this.pendingWrites.shift()
+      if (pendingWrite !== undefined) {
+        this.values[(this.readIndex + this.size) % this.capacity] = pendingWrite.value
+        this.size++
+        pendingWrite.resolve(true)
+      }
+      return { done: false, value }
+    }
+
+    if (this.failure !== undefined) {
+      throw this.failure
+    }
+
+    if (!this.open) {
+      return { done: true, value: undefined }
+    }
+
+    return new Promise((resolve, reject) => {
+      this.pendingRead = { resolve, reject }
+    })
+  }
+
+  finish() {
+    if (!this.open) {
+      return
+    }
+
+    this.open = false
+    this.pendingRead?.resolve({ done: true, value: undefined })
+    this.pendingRead = undefined
+  }
+
+  fail(error: Error) {
+    if (!this.open) {
+      return
+    }
+
+    this.failure = error
+    this.stop(error)
+  }
+
+  close() {
+    this.stop()
+  }
+
+  private stop(error?: Error) {
+    this.open = false
+    this.values.fill(undefined)
+    this.readIndex = 0
+    this.size = 0
+    for (const { resolve } of this.pendingWrites) {
+      resolve(false)
+    }
+    this.pendingWrites = []
+    if (this.pendingRead !== undefined) {
+      if (error === undefined) {
+        this.pendingRead.resolve({ done: true, value: undefined })
+      } else {
+        this.pendingRead.reject(error)
+      }
+      this.pendingRead = undefined
+    }
+  }
+}

package/node_modules/tardis-dev/src/clearcache.ts

@@ -0,0 +1,62 @@
+import { rmSync } from 'node:fs'
+import { rm } from 'node:fs/promises'
+import { debug } from './debug.ts'
+import { getOptions } from './options.ts'
+import { Filter, Exchange } from './types.ts'
+import { sha256, optimizeFilters, doubleDigit } from './handy.ts'
+
+export async function clearCache(exchange?: Exchange, filters?: Filter<any>[], year?: number, month?: number, day?: number) {
+  try {
+    const dirToRemove = getDirToRemove(exchange, filters, year, month, day)
+
+    debug('clearing cache dir: %s', dirToRemove)
+
+    await rm(dirToRemove, { force: true, recursive: true })
+
+    debug('cleared cache dir: %s', dirToRemove)
+  } catch (e) {
+    debug('clearing cache dir error: %o', e)
+  }
+}
+
+export function clearCacheSync(exchange?: Exchange, filters?: Filter<any>[], year?: number, month?: number, day?: number) {
+  try {
+    const dirToRemove = getDirToRemove(exchange, filters, year, month, day)
+
+    debug('clearing cache (sync) dir: %s', dirToRemove)
+
+    rmSync(dirToRemove, { force: true, recursive: true })
+
+    debug('cleared cache(sync) dir: %s', dirToRemove)
+  } catch (e) {
+    debug('clearing cache (sync) dir error: %o', e)
+  }
+}
+
+function getDirToRemove(exchange?: Exchange, filters?: Filter<any>[], year?: number, month?: number, day?: number) {
+  const options = getOptions()
+
+  let dirToRemove = `${options.cacheDir}/feeds`
+
+  if (exchange !== undefined) {
+    dirToRemove += `/${exchange}`
+  }
+
+  if (filters !== undefined) {
+    dirToRemove += `/${sha256(optimizeFilters(filters))}`
+  }
+
+  if (year !== undefined) {
+    dirToRemove += `/${year}`
+  }
+
+  if (month !== undefined) {
+    dirToRemove += `/${doubleDigit(month)}`
+  }
+
+  if (day !== undefined) {
+    dirToRemove += `/${doubleDigit(day)}`
+  }
+
+  return dirToRemove
+}

package/node_modules/tardis-dev/src/combine.ts

@@ -0,0 +1,134 @@
+import { closeIterator, isManagedRealTimeIterator, mergeRealTime } from './realtimeiterator.ts'
+
+type NextMessageResultWithIndex = {
+  index: number
+  result: IteratorResult<Combinable, Combinable>
+}
+
+type Combinable = { localTimestamp: Date }
+
+const DATE_MAX = new Date(8640000000000000)
+
+type OffsetMS = number | ((message: Combinable) => number)
+type IteratorPayload = AsyncIterableIterator<Combinable>[] | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS }[]
+type CombinedMessage<T> = T extends AsyncIterableIterator<infer U>[]
+  ? U
+  : T extends { stream: AsyncIterableIterator<infer U> }[]
+    ? U
+    : never
+
+async function nextWithIndex(
+  iterator: AsyncIterableIterator<Combinable> | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS },
+  index: number
+): Promise<NextMessageResultWithIndex> {
+  if ('offsetMS' in iterator) {
+    const result = await iterator.stream.next()
+
+    if (!result.done) {
+      const offsetMS = typeof iterator.offsetMS === 'function' ? iterator.offsetMS(result.value) : iterator.offsetMS
+
+      if (offsetMS !== 0) {
+        result.value.localTimestamp.setUTCMilliseconds(result.value.localTimestamp.getUTCMilliseconds() + offsetMS)
+      }
+    }
+
+    return {
+      result,
+      index
+    }
+  } else {
+    const result = await iterator.next()
+
+    return {
+      result,
+      index
+    }
+  }
+}
+
+function findOldestResult(oldest: NextMessageResultWithIndex, current: NextMessageResultWithIndex) {
+  if (oldest.result.done) {
+    return oldest
+  }
+
+  if (current.result.done) {
+    return current
+  }
+
+  const currentTimestamp = current.result.value.localTimestamp.valueOf()
+  const oldestTimestamp = oldest.result.value.localTimestamp.valueOf()
+
+  if (currentTimestamp < oldestTimestamp) {
+    return current
+  }
+
+  if (currentTimestamp === oldestTimestamp) {
+    const currentTimestampMicroSeconds = current.result.value.localTimestamp.μs || 0
+    const oldestTimestampMicroSeconds = oldest.result.value.localTimestamp.μs || 0
+
+    if (currentTimestampMicroSeconds < oldestTimestampMicroSeconds) {
+      return current
+    }
+  }
+
+  return oldest
+}
+
+// combines multiple iterators from for example multiple exchanges
+// works both for real-time and historical data
+export function combine<T extends IteratorPayload>(...iteratorsPayload: T): AsyncIterableIterator<CombinedMessage<T>> {
+  const iterators = iteratorsPayload.map((payload) => {
+    if ('stream' in payload) {
+      return payload.stream
+    }
+    return payload
+  })
+  if (isManagedRealTimeIterator(iterators[0])) {
+    return mergeRealTime(iterators) as AsyncIterableIterator<CombinedMessage<T>>
+  }
+
+  return combineHistorical(iteratorsPayload) as AsyncIterableIterator<CombinedMessage<T>>
+}
+
+async function* combineHistorical(
+  iterators: AsyncIterableIterator<Combinable>[] | { stream: AsyncIterableIterator<Combinable>; offsetMS: OffsetMS }[]
+) {
+  if (iterators.length === 0) {
+    return
+  }
+
+  try {
+    // wait for all results to resolve
+    const results = await Promise.all(iterators.map(nextWithIndex))
+    let aliveIteratorsCount = results.length
+    do {
+      // if we're dealing with historical data replay
+      // and need to return combined messages iterable sorted by local timestamp in ascending order
+
+      // find resolved one that is the 'oldest'
+      const oldestResult = results.reduce(findOldestResult, results[0])
+      const { result, index } = oldestResult
+
+      if (result.done) {
+        aliveIteratorsCount--
+
+        // we don't want finished iterators to every be considered 'oldest' again
+        // hence provide them with result that has local timestamp set to DATE_MAX
+        // and that is not done
+
+        results[index].result = {
+          done: false,
+          value: {
+            localTimestamp: DATE_MAX
+          }
+        }
+      } else {
+        // yield oldest value and replace with next value from iterable for given index
+        yield result.value
+        results[index] = await nextWithIndex(iterators[index], index)
+      }
+    } while (aliveIteratorsCount > 0)
+  } finally {
+    await Promise.all(iterators.map((iterator) => closeIterator('stream' in iterator ? iterator.stream : iterator)))
+  }
+}

package/node_modules/tardis-dev/src/computable/booksnapshot.ts

@@ -0,0 +1,327 @@
+import { decimalPlaces } from '../handy.ts'
+import { OrderBook, OnLevelRemovedCB } from '../orderbook.ts'
+import { BookChange, BookPriceLevel, BookSnapshot, Optional } from '../types.ts'
+import { Computable } from './computable.ts'
+import { ComputableContext, registerContextualFactory } from './context.ts'
+
+type BookSnapshotComputableOptions = {
+  name?: string
+  depth: number
+  grouping?: number
+  interval: number
+  removeCrossedLevels?: boolean
+  onCrossedLevelRemoved?: OnLevelRemovedCB
+}
+
+export const computeBookSnapshots = (options: BookSnapshotComputableOptions): (() => Computable<BookSnapshot>) => {
+  const factory = () => new BookSnapshotComputable(options)
+  registerContextualFactory(factory, (context) => new BookSnapshotComputable(options, context))
+  return factory
+}
+
+type OrderBookHandle = {
+  orderBook: OrderBook
+  isOwner: boolean
+}
+
+const sharedOrderBooksKey = Symbol('bookSnapshotOrderBooks')
+
+const emptyBookLevel = {
+  price: undefined,
+  amount: undefined
+}
+
+const levelsChanged = (level1: Optional<BookPriceLevel>, level2: Optional<BookPriceLevel>) => {
+  if (level1.amount !== level2.amount) {
+    return true
+  }
+
+  if (level1.price !== level2.price) {
+    return true
+  }
+
+  return false
+}
+
+class BookSnapshotComputable implements Computable<BookSnapshot> {
+  public readonly sourceDataTypes = ['book_change']
+  private _bookChanged = false
+  private _initialized = false
+
+  private readonly _type = 'book_snapshot'
+  private readonly _orderBook: OrderBook
+  private readonly _ownsOrderBook: boolean
+  private readonly _depth: number
+  private readonly _interval: number
+  private readonly _name: string
+  private readonly _grouping: number | undefined
+  private readonly _groupingDecimalPlaces: number | undefined
+  private readonly _removeCrossedLevels: boolean
+
+  private _lastUpdateTimestamp: Date = new Date(-1)
+  private _bids: Optional<BookPriceLevel>[] = []
+  private _asks: Optional<BookPriceLevel>[] = []
+  private _groupedBidBoundary: number | undefined
+  private _groupedAskBoundary: number | undefined
+
+  constructor(
+    { depth, name, interval, removeCrossedLevels, grouping, onCrossedLevelRemoved }: BookSnapshotComputableOptions,
+    context?: ComputableContext
+  ) {
+    this._depth = depth
+    this._interval = interval
+    this._grouping = grouping
+    this._groupingDecimalPlaces = this._grouping ? decimalPlaces(this._grouping) : undefined
+    this._removeCrossedLevels = Boolean(removeCrossedLevels)
+
+    const orderBookHandle = getOrderBookHandle(context, removeCrossedLevels, onCrossedLevelRemoved)
+    this._orderBook = orderBookHandle.orderBook
+    this._ownsOrderBook = orderBookHandle.isOwner
+
+    // initialize all bids/asks levels to empty ones
+    for (let i = 0; i < this._depth; i++) {
+      this._bids[i] = emptyBookLevel
+      this._asks[i] = emptyBookLevel
+    }
+
+    if (name === undefined) {
+      this._name = `${this._type}_${depth}${this._grouping ? `_grouped${this._grouping}` : ''}_${interval}ms`
+    } else {
+      this._name = name
+    }
+  }
+
+  public *compute(bookChange: BookChange) {
+    if (this._hasNewSnapshot(bookChange.timestamp)) {
+      yield this._getSnapshot(bookChange)
+    }
+
+    this._update(bookChange)
+
+    // check again after the update as book snapshot with interval set to 0 (real-time) could have changed
+    // or it's initial snapshot
+    if (this._hasNewSnapshot(bookChange.timestamp)) {
+      yield this._getSnapshot(bookChange)
+
+      if (this._initialized === false) {
+        this._initialized = true
+      }
+    }
+  }
+
+  public _hasNewSnapshot(timestamp: Date): boolean {
+    if (this._bookChanged === false) {
+      return false
+    }
+
+    // report new snapshot anytime book changed
+    if (this._interval === 0) {
+      return true
+    }
+
+    // report new snapshot for book snapshots with interval for initial snapshot
+    if (this._initialized === false) {
+      return true
+    }
+
+    const currentTimestampTimeBucket = this._getTimeBucket(timestamp)
+    const snapshotTimestampBucket = this._getTimeBucket(this._lastUpdateTimestamp)
+
+    if (currentTimestampTimeBucket > snapshotTimestampBucket) {
+      // set  timestamp to end of snapshot 'interval' period
+      this._lastUpdateTimestamp = new Date((snapshotTimestampBucket + 1) * this._interval)
+
+      return true
+    }
+
+    return false
+  }
+
+  public _update(bookChange: BookChange) {
+    if (this._ownsOrderBook) {
+      this._orderBook.update(bookChange)
+    }
+
+    if (this._grouping !== undefined) {
+      const updateBothGroupedSides = bookChange.isSnapshot || this._removeCrossedLevels
+      if (updateBothGroupedSides || this._groupedSideMayHaveChanged(bookChange.bids, this._groupedBidBoundary, false)) {
+        this._groupedBidBoundary = this._updateSideGrouped(this._orderBook.bids(), this._bids, false)
+      }
+      if (updateBothGroupedSides || this._groupedSideMayHaveChanged(bookChange.asks, this._groupedAskBoundary, true)) {
+        this._groupedAskBoundary = this._updateSideGrouped(this._orderBook.asks(), this._asks, true)
+      }
+    } else {
+      this._updatedNotGrouped()
+    }
+
+    this._lastUpdateTimestamp = bookChange.timestamp
+  }
+
+  private _groupedSideMayHaveChanged(changes: BookPriceLevel[], boundary: number | undefined, roundUp: boolean) {
+    if (changes.length === 0) return false
+    if (boundary === undefined) return true
+
+    for (const level of changes) {
+      const groupedPrice = this._getGroupedPrice(level.price, roundUp)
+      if (roundUp ? groupedPrice <= boundary : groupedPrice >= boundary) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  private _updatedNotGrouped() {
+    const bidsIterable = this._orderBook.bids()
+    const asksIterable = this._orderBook.asks()
+
+    for (let i = 0; i < this._depth; i++) {
+      const bidLevelResult = bidsIterable.next()
+      const newBid = bidLevelResult.done ? emptyBookLevel : bidLevelResult.value
+
+      if (levelsChanged(this._bids[i], newBid)) {
+        this._bids[i] = { price: newBid.price, amount: newBid.amount }
+        this._bookChanged = true
+      }
+
+      const askLevelResult = asksIterable.next()
+      const newAsk = askLevelResult.done ? emptyBookLevel : askLevelResult.value
+
+      if (levelsChanged(this._asks[i], newAsk)) {
+        this._asks[i] = { price: newAsk.price, amount: newAsk.amount }
+        this._bookChanged = true
+      }
+    }
+  }
+
+  private _getGroupedPrice(price: number, roundUp: boolean) {
+    if (this._groupingDecimalPlaces === 0) {
+      const remainder = price % this._grouping!
+      return price - remainder + (roundUp && remainder > 0 ? this._grouping! : 0)
+    }
+
+    const pow = Math.pow(10, this._groupingDecimalPlaces!)
+    const pricePow = price * pow
+    const groupPow = this._grouping! * pow
+    const remainder = (pricePow % groupPow) / pow
+
+    return (pricePow - remainder * pow + (roundUp && remainder > 0 ? groupPow : 0)) / pow
+  }
+
+  private _updateSideGrouped(
+    newLevels: IterableIterator<BookPriceLevel>,
+    existingGroupedLevels: Optional<BookPriceLevel>[],
+    roundUp: boolean
+  ) {
+    let currentGroupedPrice: number | undefined = undefined
+    let aggAmount = 0
+    let currentDepth = 0
+
+    for (const notGroupedLevel of newLevels) {
+      const groupedPrice = this._getGroupedPrice(notGroupedLevel.price, roundUp)
+
+      if (currentGroupedPrice == undefined) {
+        currentGroupedPrice = groupedPrice
+      }
+
+      if (currentGroupedPrice != groupedPrice) {
+        const existingGroupedLevel = existingGroupedLevels[currentDepth]
+        if (existingGroupedLevel.amount !== aggAmount || existingGroupedLevel.price !== currentGroupedPrice) {
+          existingGroupedLevels[currentDepth] = {
+            price: currentGroupedPrice,
+            amount: aggAmount
+          }
+          this._bookChanged = true
+        }
+
+        currentDepth++
+
+        if (currentDepth === this._depth) {
+          return currentGroupedPrice
+        }
+
+        currentGroupedPrice = groupedPrice
+        aggAmount = 0
+      }
+
+      aggAmount += notGroupedLevel.amount
+    }
+
+    if (currentDepth < this._depth && aggAmount > 0) {
+      const existingGroupedLevel = existingGroupedLevels[currentDepth]
+      if (existingGroupedLevel.amount !== aggAmount || existingGroupedLevel.price !== currentGroupedPrice) {
+        existingGroupedLevels[currentDepth] = {
+          price: currentGroupedPrice,
+          amount: aggAmount
+        }
+        this._bookChanged = true
+      }
+      currentDepth++
+    }
+
+    for (let i = currentDepth; i < this._depth; i++) {
+      if (levelsChanged(existingGroupedLevels[i], emptyBookLevel)) {
+        existingGroupedLevels[i] = emptyBookLevel
+        this._bookChanged = true
+      }
+    }
+
+    return currentDepth === this._depth ? currentGroupedPrice : undefined
+  }
+
+  public _getSnapshot(bookChange: BookChange) {
+    const snapshot: BookSnapshot = {
+      type: this._type as any,
+      symbol: bookChange.symbol,
+      exchange: bookChange.exchange,
+      name: this._name,
+      depth: this._depth,
+      interval: this._interval,
+      grouping: this._grouping,
+      bids: [...this._bids],
+      asks: [...this._asks],
+      timestamp: this._lastUpdateTimestamp,
+      localTimestamp: bookChange.localTimestamp
+    }
+
+    this._bookChanged = false
+
+    return snapshot
+  }
+
+  private _getTimeBucket(timestamp: Date) {
+    return Math.floor(timestamp.valueOf() / this._interval)
+  }
+}
+
+function getOrderBookHandle(
+  context: ComputableContext | undefined,
+  removeCrossedLevels: boolean | undefined,
+  onCrossedLevelRemoved: OnLevelRemovedCB | undefined
+): OrderBookHandle {
+  if (context === undefined) {
+    return {
+      orderBook: new OrderBook({ removeCrossedLevels, onCrossedLevelRemoved }),
+      isOwner: true
+    }
+  }
+
+  let sharedOrderBooks = context.get(sharedOrderBooksKey) as Map<boolean, OrderBook> | undefined
+  if (sharedOrderBooks === undefined) {
+    sharedOrderBooks = new Map()
+    context.set(sharedOrderBooksKey, sharedOrderBooks)
+  }
+
+  const key = removeCrossedLevels === true
+  const sharedOrderBook = sharedOrderBooks.get(key)
+  if (sharedOrderBook !== undefined) {
+    if (onCrossedLevelRemoved !== undefined) {
+      sharedOrderBook.addOnCrossedLevelRemovedCallback(onCrossedLevelRemoved)
+    }
+    return { orderBook: sharedOrderBook, isOwner: false }
+  }
+
+  const orderBook = new OrderBook({ removeCrossedLevels, onCrossedLevelRemoved })
+  sharedOrderBooks.set(key, orderBook)
+  return { orderBook, isOwner: true }
+}