tardis-machine 18.0.0 → 18.1.0

package/node_modules/tardis-dev/src/handy.ts

@@ -1,14 +1,13 @@
 import crypto, { createHash } from 'crypto'
 import { createWriteStream, mkdirSync, rmSync } from 'node:fs'
 import { rename } from 'node:fs/promises'
-import { Agent as HttpAgent } from 'node:http'
-import { Agent as HttpsAgent, type RequestOptions } from 'node:https'
-import followRedirects from 'follow-redirects'
+import { Agent as HttpAgent, request as httpRequest, type IncomingMessage } from 'node:http'
+import { Agent as HttpsAgent, request as httpsRequest, type RequestOptions } from 'node:https'
+import { pipeline } from 'node:stream/promises'
 import path from 'path'
 import { debug } from './debug.ts'
 import { Mapper } from './mappers/index.ts'
 import { Disconnect, Exchange, Filter, FilterForExchange } from './types.ts'
-const { http: followRedirectsHttp, https: followRedirectsHttps } = followRedirects
 
 export function parseAsUTCDate(val: string) {
   // Treat date-only and minute-level strings as UTC instead of local time.
@@ -456,44 +455,75 @@ async function requestViaFetch(method: string, url: string, options: HttpRequest
   }
 }
 
-async function requestViaProxy(method: string, url: string, options: HttpRequestOptions): Promise<HttpResponse> {
-  const requestClient = new URL(url).protocol === 'http:' ? followRedirectsHttp : followRedirectsHttps
-  const preparedRequest = prepareRequest(method, options)
+type DownloadRequestOptions = {
+  headers: Record<string, string>
+  timeout: number
+}
 
-  return await new Promise<HttpResponse>((resolve, reject) => {
-    const request = requestClient
-      .request(
-        url,
-        {
-          method,
-          agent: getProxyAgent(url),
-          headers: preparedRequest.headers,
-          timeout: options.timeout
-        },
-        (response) => {
-          response.setEncoding('utf8')
-          let body = ''
-          response.on('error', reject)
-          response.on('data', (chunk) => (body += chunk))
-          response.on('end', () => {
-            resolve(createHttpResponse(response.statusCode ?? 0, parseNodeResponseHeaders(response.headers), body))
-          })
-        }
-      )
-      .on('error', reject)
-      .on('timeout', () => {
-        reject(new Error('Request timed out'))
-        request.destroy()
-      })
-
-    if (preparedRequest.body !== undefined) {
-      request.write(preparedRequest.body)
-    }
+function sendHttpRequest(url: string | URL, options: RequestOptions, body?: string): Promise<IncomingMessage> {
+  const requestUrl = new URL(url)
+  const requestClient = requestUrl.protocol === 'http:' ? httpRequest : httpsRequest
+  const agent = getProxyAgent(requestUrl) ?? (requestUrl.protocol === 'https:' ? httpsAgent : undefined)
+
+  return new Promise((resolve, reject) => {
+    const request = requestClient(requestUrl, { ...options, agent }, resolve)
 
+    request.once('error', reject)
+    request.once('timeout', () => request.destroy(new Error('Request timed out')))
+    if (body !== undefined) {
+      request.write(body)
+    }
     request.end()
   })
 }
 
+async function openDownloadResponse(url: string, options: DownloadRequestOptions) {
+  const response = await sendHttpRequest(url, options)
+  const statusCode = response.statusCode ?? 0
+  const location = response.headers.location
+  if (location === undefined || statusCode < 300 || statusCode >= 400) {
+    return response
+  }
+
+  response.destroy()
+  const sourceUrl = new URL(url)
+  const redirectUrl = new URL(location, sourceUrl)
+  const headers: Record<string, string> = { ...options.headers }
+  if (sourceUrl.origin !== redirectUrl.origin) {
+    delete headers.Authorization
+    delete headers.authorization
+  }
+
+  return sendHttpRequest(redirectUrl, {
+    ...options,
+    headers
+  })
+}
+
+async function readResponseText(response: IncomingMessage) {
+  response.setEncoding('utf8')
+  let body = ''
+  for await (const chunk of response) {
+    body += chunk
+  }
+  return body
+}
+
+async function requestViaProxy(method: string, url: string, options: HttpRequestOptions): Promise<HttpResponse> {
+  const preparedRequest = prepareRequest(method, options)
+  const response = await sendHttpRequest(
+    url,
+    {
+      method,
+      headers: preparedRequest.headers,
+      timeout: options.timeout
+    },
+    preparedRequest.body
+  )
+
+  return createHttpResponse(response.statusCode ?? 0, parseNodeResponseHeaders(response.headers), await readResponseText(response))
+}
+
 async function request(method: string, url: string, options: HttpRequestOptions = {}) {
   const retrySettings = getRetrySettings(method, options.retry)
 
@@ -551,7 +581,8 @@ export async function download({
   url,
   userAgent,
   appendContentEncodingExtension = false,
-  acceptEncoding = 'gzip'
+  acceptEncoding = 'gzip',
+  attemptTimeoutMS = 90 * ONE_SEC_IN_MS
 }: {
   url: string
   downloadPath: string
@@ -559,9 +590,10 @@ export async function download({
   apiKey: string
   appendContentEncodingExtension?: boolean
   acceptEncoding?: string
+  attemptTimeoutMS?: number
 }) {
   const httpRequestOptions = {
-    timeout: 90 * ONE_SEC_IN_MS,
+    timeout: attemptTimeoutMS,
     headers: {
       'Accept-Encoding': acceptEncoding,
       'User-Agent': userAgent,
@@ -622,17 +654,22 @@ export function cleanTempFiles() {
   tmpFileCleanups.forEach((cleanup) => cleanup())
 }
 
-async function _downloadFile(requestOptions: RequestOptions, url: string, downloadPath: string, appendContentEncodingExtension: boolean) {
+async function _downloadFile(
+  requestOptions: DownloadRequestOptions,
+  url: string,
+  downloadPath: string,
+  appendContentEncodingExtension: boolean
+) {
   // first ensure that directory where we want to download file exists
   mkdirSync(path.dirname(downloadPath), { recursive: true })
 
   // create write file stream that we'll write data into - first as unconfirmed temp file
 
   const tmpFilePath = `${downloadPath}${crypto.randomBytes(8).toString('hex')}.unconfirmed`
-  const fileWriteStream = createWriteStream(tmpFilePath)
+  let fileWriteStream: ReturnType<typeof createWriteStream> | undefined
   const cleanup = () => {
     try {
-      fileWriteStream.destroy()
+      fileWriteStream?.destroy()
       rmSync(tmpFilePath, { force: true })
     } catch {}
   }
@@ -642,59 +679,29 @@ async function _downloadFile(requestOptions: RequestOptions, url: string, downlo
 
   try {
     // based on https://github.com/nodejs/node/issues/28172 - only reliable way to consume response stream and avoiding all the 'gotchas'
-    let responseHeaders: Record<string, string> = {}
-    await new Promise<void>((resolve, reject) => {
-      const protocol = new URL(url).protocol
-      const requestClient = protocol === 'http:' ? followRedirectsHttp : followRedirectsHttps
-      const agent = getProxyAgent(url) ?? (protocol === 'https:' ? httpsAgent : undefined)
-      const req = requestClient
-        .get(url, { ...requestOptions, agent }, (res) => {
-          const { statusCode } = res
-          if (statusCode !== 200) {
-            // read the error response text and throw it as an HttpError
-            res.setEncoding('utf8')
-            let body = ''
-            res.on('error', reject)
-            res.on('data', (chunk) => (body += chunk))
-            res.on('end', () => {
-              reject(new HttpError(statusCode!, body, url))
-            })
-          } else {
-            responseHeaders = parseNodeResponseHeaders(res.headers)
-            if (appendContentEncodingExtension) {
-              const contentEncoding = asSingleHeaderValue(res.headers['content-encoding'])
-              if (contentEncoding === 'zstd') {
-                finalDownloadPath = `${downloadPath}.zst`
-              } else if (contentEncoding === undefined || contentEncoding === 'gzip') {
-                finalDownloadPath = `${downloadPath}.gz`
-              } else {
-                reject(new Error(`Unsupported data feed content encoding: ${contentEncoding}`))
-                return
-              }
-            }
+    const response = await openDownloadResponse(url, requestOptions)
+    const { statusCode } = response
+    if (statusCode !== 200) {
+      throw new HttpError(statusCode ?? 0, await readResponseText(response), url)
+    }
 
-            // consume the response stream by writing it to the file
-            res
-              .on('error', reject)
-              .on('aborted', () => reject(new Error('Request aborted')))
-              .pipe(fileWriteStream)
-              .on('error', reject)
-              .on('finish', () => {
-                if (res.complete) {
-                  resolve()
-                } else {
-                  reject(new Error('The connection was terminated while the message was still being sent'))
-                }
-              })
-          }
-        })
-        .on('error', reject)
-        .on('timeout', () => {
-          debug('download file request timeout, %s', url)
-          reject(new Error('Request timed out'))
-          req.destroy()
-        })
-    })
+    const responseHeaders = parseNodeResponseHeaders(response.headers)
+    if (appendContentEncodingExtension) {
+      const contentEncoding = asSingleHeaderValue(response.headers['content-encoding'])
+      if (contentEncoding === 'zstd') {
+        finalDownloadPath = `${downloadPath}.zst`
+      } else if (contentEncoding === undefined || contentEncoding === 'gzip') {
+        finalDownloadPath = `${downloadPath}.gz`
+      } else {
+        throw new Error(`Unsupported data feed content encoding: ${contentEncoding}`)
+      }
+    }
+
+    fileWriteStream = createWriteStream(tmpFilePath)
+    await pipeline(response, fileWriteStream)
+    if (!response.complete) {
+      throw new Error('The connection was terminated while the message was still being sent')
+    }
 
     // finally when saving from the network to file has succeded, rename tmp file to normal name
     // then we're sure that responses is 100% saved and also even if different process was doing the same we're good

package/node_modules/tardis-dev/src/mappers/hyperliquid.ts

@@ -1,11 +1,16 @@
 import { BookChange, BookTicker, DerivativeTicker, Trade } from '../types.ts'
 import { Mapper, PendingTickerInfoHelper } from './mapper.ts'
-import { exchangeMappers } from './registry.ts'
+import { exchangeMappers, mapper } from './registry.ts'
+
+const HYPERLIQUID_FAST_BOOK_SWITCH_DATE = new Date('2026-06-17T00:00:00.000Z')
 
 export const hyperliquidMappers = exchangeMappers({
   hyperliquid: {
     trades: () => new HyperliquidTradesMapper(),
-    bookChanges: () => new HyperliquidBookChangeMapper(),
+    bookChanges: mapper([
+      { until: HYPERLIQUID_FAST_BOOK_SWITCH_DATE, use: () => new HyperliquidBookChangeMapper('l2Book') },
+      { use: () => new HyperliquidBookChangeMapper('fastBook') }
+    ]),
     derivativeTickers: () => new HyperliquidDerivativeTickerMapper(),
     bookTickers: () => new HyperliquidBookTickerMapper()
   }
@@ -78,8 +83,14 @@ function mapHyperliquidLevel(level: HyperliquidWsLevel) {
   }
 }
 class HyperliquidBookChangeMapper implements Mapper<'hyperliquid', BookChange> {
+  constructor(private readonly filterChannel: HyperliquidBookChannel) {}
+
   canHandle(message: HyperliquidWsBookMessage) {
-    return message.channel === 'l2Book'
+    // Hyperliquid returns channel=l2Book for both live feeds and marks only the fast feed with data.fast=true.
+    // The recorder rewrites that channel to fastBook so historical data can distinguish the two feeds.
+    const isFastBook = message.channel === 'fastBook' || (message.channel === 'l2Book' && message.data.fast === true)
+
+    return this.filterChannel === 'fastBook' ? isFastBook : message.channel === 'l2Book' && isFastBook === false
   }
 
   getFilters(symbols?: string[]) {
@@ -87,7 +98,7 @@ class HyperliquidBookChangeMapper implements Mapper<'hyperliquid', BookChange> {
 
     return [
       {
-        channel: 'l2Book',
+        channel: this.filterChannel,
         symbols
       }
     ]
@@ -206,14 +217,17 @@ type HyperliquidTradeMessage = {
 }
 
 type HyperliquidWsBookMessage = {
-  channel: 'l2Book'
+  channel: HyperliquidBookChannel
   data: {
     coin: 'ATOM'
     time: 1730160007687
     levels: [HyperliquidWsLevel[], HyperliquidWsLevel[]]
+    fast?: boolean
   }
 }
 
+type HyperliquidBookChannel = 'l2Book' | 'fastBook'
+
 type HyperliquidWsLevel = {
   px: string // price
   sz: string // size

package/node_modules/tardis-dev/src/realtimefeeds/mexc.ts

@@ -1,71 +1,13 @@
-import protobuf from 'protobufjs'
 import { CircularBuffer, getJSON, wait } from '../handy.ts'
 import type { MexcDepthSnapshotMessage } from '../mappers/mexc.ts'
 import { Filter } from '../types.ts'
 import { RealTimeFeedBase } from './realtimefeed.ts'
+import { decodeMexcProtobufMessage } from './mexcprotobuf.ts'
 
 export class MexcRealTimeFeed extends RealTimeFeedBase {
   private static readonly depthChannel = 'spot@public.aggre.depth.v3.api.pb@10ms'
   private static readonly depthSnapshotChannel = 'spot@public.aggre.depth.snapshot.v3.api.pb@10ms'
   private static readonly jsonObjectStart = '{'.charCodeAt(0)
-  private static pushDataV3ApiWrapper: protobuf.Type | undefined
-  /**
-   * MEXC protobuf docs at @see https://www.mexc.com/api-docs/spot-v3/websocket-market-streams/protocol-buffers-integration
-   * and schemas from @see https://github.com/mexcdevelop/websocket-proto, mirrored in tardis-recorder/datafeeds/src/mexc/proto.
-   */
-  private static readonly pushDataV3ApiWrapperSchema = `
-    syntax = "proto3";
-
-    message PushDataV3ApiWrapper {
-      string channel = 1;
-      oneof body {
-        PublicAggreDepthsV3Api publicAggreDepths = 313;
-        PublicAggreDealsV3Api publicAggreDeals = 314;
-        PublicAggreBookTickerV3Api publicAggreBookTicker = 315;
-      }
-
-      optional string symbol = 3;
-      optional string symbolId = 4;
-      optional int64 createTime = 5;
-      optional int64 sendTime = 6;
-    }
-
-    message PublicAggreDealsV3Api {
-      repeated PublicAggreDealsV3ApiItem deals = 1;
-      string eventType = 2;
-    }
-
-    message PublicAggreDealsV3ApiItem {
-      string price = 1;
-      string quantity = 2;
-      int32 tradeType = 3;
-      int64 time = 4;
-      string tradeId = 5;
-    }
-
-    message PublicAggreDepthsV3Api {
-      repeated PublicAggreDepthV3ApiItem asks = 1;
-      repeated PublicAggreDepthV3ApiItem bids = 2;
-      string eventType = 3;
-      string fromVersion = 4;
-      string toVersion = 5;
-      int64 lastOrderCreateTime = 6;
-    }
-
-    message PublicAggreDepthV3ApiItem {
-      string price = 1;
-      string quantity = 2;
-    }
-
-    message PublicAggreBookTickerV3Api {
-      string bidPrice = 1;
-      string bidQuantity = 2;
-      string askPrice = 3;
-      string askQuantity = 4;
-      string version = 5;
-      int64 lastOrderCreateTime = 6;
-    }
-  `
 
   private readonly pendingDepthSnapshotSymbols = new Set<string>()
   private readonly bufferedDepthUpdates = new Map<string, CircularBuffer<MexcDepthUpdateData>>()
@@ -114,11 +56,7 @@ export class MexcRealTimeFeed extends RealTimeFeedBase {
       return JSON.parse(buffer.toString())
     }
 
-    const pushDataV3ApiWrapper = this.getPushDataV3ApiWrapper()
-    return pushDataV3ApiWrapper.toObject(pushDataV3ApiWrapper.decode(buffer), {
-      longs: String,
-      arrays: true
-    })
+    return decodeMexcProtobufMessage(buffer)
   }
 
   protected override messageIsError(message: any): boolean {
@@ -168,14 +106,6 @@ export class MexcRealTimeFeed extends RealTimeFeedBase {
     this.send({ method: 'PING' })
   }
 
-  private getPushDataV3ApiWrapper() {
-    MexcRealTimeFeed.pushDataV3ApiWrapper ??= protobuf
-      .parse(MexcRealTimeFeed.pushDataV3ApiWrapperSchema)
-      .root.lookupType('PushDataV3ApiWrapper')
-
-    return MexcRealTimeFeed.pushDataV3ApiWrapper
-  }
-
   private resetDepthSnapshotTracking(filters: Required<Filter<string>>[]) {
     this.pendingDepthSnapshotSymbols.clear()
     this.bufferedDepthUpdates.clear()