tardis-machine 18.0.0 → 18.0.1

package/node_modules/tardis-dev/src/handy.ts

@@ -1,14 +1,13 @@
 import crypto, { createHash } from 'crypto'
 import { createWriteStream, mkdirSync, rmSync } from 'node:fs'
 import { rename } from 'node:fs/promises'
-import { Agent as HttpAgent } from 'node:http'
-import { Agent as HttpsAgent, type RequestOptions } from 'node:https'
-import followRedirects from 'follow-redirects'
+import { Agent as HttpAgent, request as httpRequest, type IncomingMessage } from 'node:http'
+import { Agent as HttpsAgent, request as httpsRequest, type RequestOptions } from 'node:https'
+import { pipeline } from 'node:stream/promises'
 import path from 'path'
 import { debug } from './debug.ts'
 import { Mapper } from './mappers/index.ts'
 import { Disconnect, Exchange, Filter, FilterForExchange } from './types.ts'
-const { http: followRedirectsHttp, https: followRedirectsHttps } = followRedirects
 
 export function parseAsUTCDate(val: string) {
   // Treat date-only and minute-level strings as UTC instead of local time.
@@ -456,44 +455,75 @@ async function requestViaFetch(method: string, url: string, options: HttpRequest
   }
 }
 
-async function requestViaProxy(method: string, url: string, options: HttpRequestOptions): Promise<HttpResponse> {
-  const requestClient = new URL(url).protocol === 'http:' ? followRedirectsHttp : followRedirectsHttps
-  const preparedRequest = prepareRequest(method, options)
+type DownloadRequestOptions = {
+  headers: Record<string, string>
+  timeout: number
+}
 
-  return await new Promise<HttpResponse>((resolve, reject) => {
-    const request = requestClient
-      .request(
-        url,
-        {
-          method,
-          agent: getProxyAgent(url),
-          headers: preparedRequest.headers,
-          timeout: options.timeout
-        },
-        (response) => {
-          response.setEncoding('utf8')
-          let body = ''
-          response.on('error', reject)
-          response.on('data', (chunk) => (body += chunk))
-          response.on('end', () => {
-            resolve(createHttpResponse(response.statusCode ?? 0, parseNodeResponseHeaders(response.headers), body))
-          })
-        }
-      )
-      .on('error', reject)
-      .on('timeout', () => {
-        reject(new Error('Request timed out'))
-        request.destroy()
-      })
-
-    if (preparedRequest.body !== undefined) {
-      request.write(preparedRequest.body)
-    }
+function sendHttpRequest(url: string | URL, options: RequestOptions, body?: string): Promise<IncomingMessage> {
+  const requestUrl = new URL(url)
+  const requestClient = requestUrl.protocol === 'http:' ? httpRequest : httpsRequest
+  const agent = getProxyAgent(requestUrl) ?? (requestUrl.protocol === 'https:' ? httpsAgent : undefined)
+
+  return new Promise((resolve, reject) => {
+    const request = requestClient(requestUrl, { ...options, agent }, resolve)
 
+    request.once('error', reject)
+    request.once('timeout', () => request.destroy(new Error('Request timed out')))
+    if (body !== undefined) {
+      request.write(body)
+    }
     request.end()
   })
 }
 
+async function openDownloadResponse(url: string, options: DownloadRequestOptions) {
+  const response = await sendHttpRequest(url, options)
+  const statusCode = response.statusCode ?? 0
+  const location = response.headers.location
+  if (location === undefined || statusCode < 300 || statusCode >= 400) {
+    return response
+  }
+
+  response.destroy()
+  const sourceUrl = new URL(url)
+  const redirectUrl = new URL(location, sourceUrl)
+  const headers: Record<string, string> = { ...options.headers }
+  if (sourceUrl.origin !== redirectUrl.origin) {
+    delete headers.Authorization
+    delete headers.authorization
+  }
+
+  return sendHttpRequest(redirectUrl, {
+    ...options,
+    headers
+  })
+}
+
+async function readResponseText(response: IncomingMessage) {
+  response.setEncoding('utf8')
+  let body = ''
+  for await (const chunk of response) {
+    body += chunk
+  }
+  return body
+}
+
+async function requestViaProxy(method: string, url: string, options: HttpRequestOptions): Promise<HttpResponse> {
+  const preparedRequest = prepareRequest(method, options)
+  const response = await sendHttpRequest(
+    url,
+    {
+      method,
+      headers: preparedRequest.headers,
+      timeout: options.timeout
+    },
+    preparedRequest.body
+  )
+
+  return createHttpResponse(response.statusCode ?? 0, parseNodeResponseHeaders(response.headers), await readResponseText(response))
+}
+
 async function request(method: string, url: string, options: HttpRequestOptions = {}) {
   const retrySettings = getRetrySettings(method, options.retry)
 
@@ -622,7 +652,12 @@ export function cleanTempFiles() {
   tmpFileCleanups.forEach((cleanup) => cleanup())
 }
 
-async function _downloadFile(requestOptions: RequestOptions, url: string, downloadPath: string, appendContentEncodingExtension: boolean) {
+async function _downloadFile(
+  requestOptions: DownloadRequestOptions,
+  url: string,
+  downloadPath: string,
+  appendContentEncodingExtension: boolean
+) {
   // first ensure that directory where we want to download file exists
   mkdirSync(path.dirname(downloadPath), { recursive: true })
 
@@ -642,59 +677,28 @@ async function _downloadFile(requestOptions: RequestOptions, url: string, downlo
 
   try {
     // based on https://github.com/nodejs/node/issues/28172 - only reliable way to consume response stream and avoiding all the 'gotchas'
-    let responseHeaders: Record<string, string> = {}
-    await new Promise<void>((resolve, reject) => {
-      const protocol = new URL(url).protocol
-      const requestClient = protocol === 'http:' ? followRedirectsHttp : followRedirectsHttps
-      const agent = getProxyAgent(url) ?? (protocol === 'https:' ? httpsAgent : undefined)
-      const req = requestClient
-        .get(url, { ...requestOptions, agent }, (res) => {
-          const { statusCode } = res
-          if (statusCode !== 200) {
-            // read the error response text and throw it as an HttpError
-            res.setEncoding('utf8')
-            let body = ''
-            res.on('error', reject)
-            res.on('data', (chunk) => (body += chunk))
-            res.on('end', () => {
-              reject(new HttpError(statusCode!, body, url))
-            })
-          } else {
-            responseHeaders = parseNodeResponseHeaders(res.headers)
-            if (appendContentEncodingExtension) {
-              const contentEncoding = asSingleHeaderValue(res.headers['content-encoding'])
-              if (contentEncoding === 'zstd') {
-                finalDownloadPath = `${downloadPath}.zst`
-              } else if (contentEncoding === undefined || contentEncoding === 'gzip') {
-                finalDownloadPath = `${downloadPath}.gz`
-              } else {
-                reject(new Error(`Unsupported data feed content encoding: ${contentEncoding}`))
-                return
-              }
-            }
+    const response = await openDownloadResponse(url, requestOptions)
+    const { statusCode } = response
+    if (statusCode !== 200) {
+      throw new HttpError(statusCode ?? 0, await readResponseText(response), url)
+    }
 
-            // consume the response stream by writing it to the file
-            res
-              .on('error', reject)
-              .on('aborted', () => reject(new Error('Request aborted')))
-              .pipe(fileWriteStream)
-              .on('error', reject)
-              .on('finish', () => {
-                if (res.complete) {
-                  resolve()
-                } else {
-                  reject(new Error('The connection was terminated while the message was still being sent'))
-                }
-              })
-          }
-        })
-        .on('error', reject)
-        .on('timeout', () => {
-          debug('download file request timeout, %s', url)
-          reject(new Error('Request timed out'))
-          req.destroy()
-        })
-    })
+    const responseHeaders = parseNodeResponseHeaders(response.headers)
+    if (appendContentEncodingExtension) {
+      const contentEncoding = asSingleHeaderValue(response.headers['content-encoding'])
+      if (contentEncoding === 'zstd') {
+        finalDownloadPath = `${downloadPath}.zst`
+      } else if (contentEncoding === undefined || contentEncoding === 'gzip') {
+        finalDownloadPath = `${downloadPath}.gz`
+      } else {
+        throw new Error(`Unsupported data feed content encoding: ${contentEncoding}`)
+      }
+    }
+
+    await pipeline(response, fileWriteStream)
+    if (!response.complete) {
+      throw new Error('The connection was terminated while the message was still being sent')
+    }
 
     // finally when saving from the network to file has succeded, rename tmp file to normal name
     // then we're sure that responses is 100% saved and also even if different process was doing the same we're good

package/node_modules/tardis-dev/src/realtimefeeds/mexc.ts

@@ -1,71 +1,13 @@
-import protobuf from 'protobufjs'
 import { CircularBuffer, getJSON, wait } from '../handy.ts'
 import type { MexcDepthSnapshotMessage } from '../mappers/mexc.ts'
 import { Filter } from '../types.ts'
 import { RealTimeFeedBase } from './realtimefeed.ts'
+import { decodeMexcProtobufMessage } from './mexcprotobuf.ts'
 
 export class MexcRealTimeFeed extends RealTimeFeedBase {
   private static readonly depthChannel = 'spot@public.aggre.depth.v3.api.pb@10ms'
   private static readonly depthSnapshotChannel = 'spot@public.aggre.depth.snapshot.v3.api.pb@10ms'
   private static readonly jsonObjectStart = '{'.charCodeAt(0)
-  private static pushDataV3ApiWrapper: protobuf.Type | undefined
-  /**
-   * MEXC protobuf docs at @see https://www.mexc.com/api-docs/spot-v3/websocket-market-streams/protocol-buffers-integration
-   * and schemas from @see https://github.com/mexcdevelop/websocket-proto, mirrored in tardis-recorder/datafeeds/src/mexc/proto.
-   */
-  private static readonly pushDataV3ApiWrapperSchema = `
-    syntax = "proto3";
-
-    message PushDataV3ApiWrapper {
-      string channel = 1;
-      oneof body {
-        PublicAggreDepthsV3Api publicAggreDepths = 313;
-        PublicAggreDealsV3Api publicAggreDeals = 314;
-        PublicAggreBookTickerV3Api publicAggreBookTicker = 315;
-      }
-
-      optional string symbol = 3;
-      optional string symbolId = 4;
-      optional int64 createTime = 5;
-      optional int64 sendTime = 6;
-    }
-
-    message PublicAggreDealsV3Api {
-      repeated PublicAggreDealsV3ApiItem deals = 1;
-      string eventType = 2;
-    }
-
-    message PublicAggreDealsV3ApiItem {
-      string price = 1;
-      string quantity = 2;
-      int32 tradeType = 3;
-      int64 time = 4;
-      string tradeId = 5;
-    }
-
-    message PublicAggreDepthsV3Api {
-      repeated PublicAggreDepthV3ApiItem asks = 1;
-      repeated PublicAggreDepthV3ApiItem bids = 2;
-      string eventType = 3;
-      string fromVersion = 4;
-      string toVersion = 5;
-      int64 lastOrderCreateTime = 6;
-    }
-
-    message PublicAggreDepthV3ApiItem {
-      string price = 1;
-      string quantity = 2;
-    }
-
-    message PublicAggreBookTickerV3Api {
-      string bidPrice = 1;
-      string bidQuantity = 2;
-      string askPrice = 3;
-      string askQuantity = 4;
-      string version = 5;
-      int64 lastOrderCreateTime = 6;
-    }
-  `
 
   private readonly pendingDepthSnapshotSymbols = new Set<string>()
   private readonly bufferedDepthUpdates = new Map<string, CircularBuffer<MexcDepthUpdateData>>()
@@ -114,11 +56,7 @@ export class MexcRealTimeFeed extends RealTimeFeedBase {
       return JSON.parse(buffer.toString())
     }
 
-    const pushDataV3ApiWrapper = this.getPushDataV3ApiWrapper()
-    return pushDataV3ApiWrapper.toObject(pushDataV3ApiWrapper.decode(buffer), {
-      longs: String,
-      arrays: true
-    })
+    return decodeMexcProtobufMessage(buffer)
   }
 
   protected override messageIsError(message: any): boolean {
@@ -168,14 +106,6 @@ export class MexcRealTimeFeed extends RealTimeFeedBase {
     this.send({ method: 'PING' })
   }
 
-  private getPushDataV3ApiWrapper() {
-    MexcRealTimeFeed.pushDataV3ApiWrapper ??= protobuf
-      .parse(MexcRealTimeFeed.pushDataV3ApiWrapperSchema)
-      .root.lookupType('PushDataV3ApiWrapper')
-
-    return MexcRealTimeFeed.pushDataV3ApiWrapper
-  }
-
   private resetDepthSnapshotTracking(filters: Required<Filter<string>>[]) {
     this.pendingDepthSnapshotSymbols.clear()
     this.bufferedDepthUpdates.clear()

package/node_modules/tardis-dev/src/realtimefeeds/mexcprotobuf.ts

@@ -0,0 +1,364 @@
+const textDecoder = new TextDecoder()
+
+// Decodes only the current MEXC real-time channels supported by MexcRealTimeFeed.
+// Field numbers mirror the official schemas at commit 7b8ac7a6681f28551612a5a7cefbb7e09b56bb85:
+// https://github.com/mexcdevelop/websocket-proto/tree/7b8ac7a6681f28551612a5a7cefbb7e09b56bb85
+// int64 values remain decimal strings and repeated fields remain empty arrays when absent,
+// preserving the output shape previously produced by protobufjs.
+export function decodeMexcProtobufMessage(data: Uint8Array) {
+  const reader = new ProtobufReader(data)
+  const message: {
+    channel?: string
+    symbol?: string
+    symbolId?: string
+    createTime?: string
+    sendTime?: string
+    publicAggreDepths?: ReturnType<typeof decodeDepths>
+    publicAggreDeals?: ReturnType<typeof decodeDeals>
+    publicAggreBookTicker?: ReturnType<typeof decodeBookTicker>
+  } = {}
+
+  while (reader.done === false) {
+    const { fieldNumber, wireType } = reader.readTag()
+
+    switch (fieldNumber) {
+      case 1:
+        message.channel = reader.readString(wireType)
+        break
+      case 3:
+        message.symbol = reader.readString(wireType)
+        break
+      case 4:
+        message.symbolId = reader.readString(wireType)
+        break
+      case 5:
+        message.createTime = reader.readInt64String(wireType)
+        break
+      case 6:
+        message.sendTime = reader.readInt64String(wireType)
+        break
+      case 313:
+        message.publicAggreDepths = decodeDepths(reader.readBytes(wireType))
+        break
+      case 314:
+        message.publicAggreDeals = decodeDeals(reader.readBytes(wireType))
+        break
+      case 315:
+        message.publicAggreBookTicker = decodeBookTicker(reader.readBytes(wireType))
+        break
+      default:
+        reader.skipField(wireType)
+    }
+  }
+
+  return message
+}
+
+function decodeDeals(data: Uint8Array) {
+  const reader = new ProtobufReader(data)
+  const message: { deals: ReturnType<typeof decodeTrade>[]; eventType?: string } = { deals: [] }
+
+  while (reader.done === false) {
+    const { fieldNumber, wireType } = reader.readTag()
+
+    switch (fieldNumber) {
+      case 1:
+        message.deals.push(decodeTrade(reader.readBytes(wireType)))
+        break
+      case 2:
+        message.eventType = reader.readString(wireType)
+        break
+      default:
+        reader.skipField(wireType)
+    }
+  }
+
+  return message
+}
+
+function decodeTrade(data: Uint8Array) {
+  const reader = new ProtobufReader(data)
+  const message: { price?: string; quantity?: string; tradeType?: number; time?: string; tradeId?: string } = {}
+
+  while (reader.done === false) {
+    const { fieldNumber, wireType } = reader.readTag()
+
+    switch (fieldNumber) {
+      case 1:
+        message.price = reader.readString(wireType)
+        break
+      case 2:
+        message.quantity = reader.readString(wireType)
+        break
+      case 3:
+        message.tradeType = reader.readInt32(wireType)
+        break
+      case 4:
+        message.time = reader.readInt64String(wireType)
+        break
+      case 5:
+        message.tradeId = reader.readString(wireType)
+        break
+      default:
+        reader.skipField(wireType)
+    }
+  }
+
+  return message
+}
+
+function decodeDepths(data: Uint8Array) {
+  const reader = new ProtobufReader(data)
+  const message: {
+    asks: ReturnType<typeof decodePriceLevel>[]
+    bids: ReturnType<typeof decodePriceLevel>[]
+    eventType?: string
+    fromVersion?: string
+    toVersion?: string
+    lastOrderCreateTime?: string
+  } = { asks: [], bids: [] }
+
+  while (reader.done === false) {
+    const { fieldNumber, wireType } = reader.readTag()
+
+    switch (fieldNumber) {
+      case 1:
+        message.asks.push(decodePriceLevel(reader.readBytes(wireType)))
+        break
+      case 2:
+        message.bids.push(decodePriceLevel(reader.readBytes(wireType)))
+        break
+      case 3:
+        message.eventType = reader.readString(wireType)
+        break
+      case 4:
+        message.fromVersion = reader.readString(wireType)
+        break
+      case 5:
+        message.toVersion = reader.readString(wireType)
+        break
+      case 6:
+        message.lastOrderCreateTime = reader.readInt64String(wireType)
+        break
+      default:
+        reader.skipField(wireType)
+    }
+  }
+
+  return message
+}
+
+function decodePriceLevel(data: Uint8Array) {
+  const reader = new ProtobufReader(data)
+  const message: { price?: string; quantity?: string } = {}
+
+  while (reader.done === false) {
+    const { fieldNumber, wireType } = reader.readTag()
+
+    switch (fieldNumber) {
+      case 1:
+        message.price = reader.readString(wireType)
+        break
+      case 2:
+        message.quantity = reader.readString(wireType)
+        break
+      default:
+        reader.skipField(wireType)
+    }
+  }
+
+  return message
+}
+
+function decodeBookTicker(data: Uint8Array) {
+  const reader = new ProtobufReader(data)
+  const message: {
+    bidPrice?: string
+    bidQuantity?: string
+    askPrice?: string
+    askQuantity?: string
+    version?: string
+    lastOrderCreateTime?: string
+  } = {}
+
+  while (reader.done === false) {
+    const { fieldNumber, wireType } = reader.readTag()
+
+    switch (fieldNumber) {
+      case 1:
+        message.bidPrice = reader.readString(wireType)
+        break
+      case 2:
+        message.bidQuantity = reader.readString(wireType)
+        break
+      case 3:
+        message.askPrice = reader.readString(wireType)
+        break
+      case 4:
+        message.askQuantity = reader.readString(wireType)
+        break
+      case 5:
+        message.version = reader.readString(wireType)
+        break
+      case 6:
+        message.lastOrderCreateTime = reader.readInt64String(wireType)
+        break
+      default:
+        reader.skipField(wireType)
+    }
+  }
+
+  return message
+}
+
+class ProtobufReader {
+  private offset = 0
+
+  constructor(private readonly data: Uint8Array) {}
+
+  get done() {
+    return this.offset === this.data.length
+  }
+
+  readTag() {
+    const tag = this.readVarint()
+    if (typeof tag === 'bigint' || tag > 0xffffffff) {
+      throw new Error('Invalid protobuf tag')
+    }
+
+    const fieldNumber = tag >>> 3
+    const wireType = tag & 7
+
+    if (fieldNumber === 0) {
+      throw new Error('Invalid protobuf field number 0')
+    }
+
+    return { fieldNumber, wireType }
+  }
+
+  readString(wireType: number) {
+    this.expectWireType(wireType, 2)
+    const length = this.readLength()
+    const start = this.offset
+    const end = start + length
+
+    if (end > this.data.length) {
+      throw new Error('Invalid protobuf message length')
+    }
+
+    this.offset = end
+    return Buffer.isBuffer(this.data) ? this.data.toString('utf8', start, end) : textDecoder.decode(this.data.subarray(start, end))
+  }
+
+  readBytes(wireType: number) {
+    this.expectWireType(wireType, 2)
+    const length = this.readLength()
+    const end = this.offset + length
+
+    if (end > this.data.length) {
+      throw new Error('Invalid protobuf message length')
+    }
+
+    const value = this.data.subarray(this.offset, end)
+    this.offset = end
+    return value
+  }
+
+  readInt32(wireType: number) {
+    this.expectWireType(wireType, 0)
+    const value = this.readVarint()
+    return typeof value === 'number' ? value | 0 : Number(BigInt.asIntN(32, value))
+  }
+
+  readInt64String(wireType: number) {
+    this.expectWireType(wireType, 0)
+    const value = this.readVarint()
+    return typeof value === 'number' ? value.toString() : BigInt.asIntN(64, value).toString()
+  }
+
+  skipField(wireType: number): void {
+    switch (wireType) {
+      case 0:
+        this.skipVarint()
+        return
+      case 1:
+        this.skipBytes(8)
+        return
+      case 2:
+        this.skipBytes(this.readLength())
+        return
+      case 5:
+        this.skipBytes(4)
+        return
+      default:
+        throw new Error(`Unsupported protobuf wire type ${wireType}`)
+    }
+  }
+
+  private readVarint() {
+    let value = 0
+    let multiplier = 1
+
+    // Seven base-128 digits use at most 49 bits, so they remain exact and fast as a number.
+    for (let index = 0; index < 7; index++) {
+      const byte = this.readByte()
+      value += (byte & 0x7f) * multiplier
+
+      if ((byte & 0x80) === 0) {
+        return value
+      }
+      multiplier *= 0x80
+    }
+
+    // Larger int64 values need BigInt to preserve every bit before converting them to strings.
+    let bigValue = BigInt(value)
+    for (let index = 7; index < 10; index++) {
+      const byte = this.readByte()
+      bigValue |= BigInt(byte & 0x7f) << BigInt(index * 7)
+
+      if ((byte & 0x80) === 0) {
+        return bigValue
+      }
+    }
+
+    throw new Error('Invalid protobuf varint')
+  }
+
+  private readLength() {
+    const length = this.readVarint()
+    if (typeof length === 'bigint' || Number.isSafeInteger(length) === false) {
+      throw new Error('Invalid protobuf message length')
+    }
+    return length
+  }
+
+  private skipVarint() {
+    for (let index = 0; index < 10; index++) {
+      if ((this.readByte() & 0x80) === 0) {
+        return
+      }
+    }
+    throw new Error('Invalid protobuf varint')
+  }
+
+  private readByte() {
+    if (this.offset >= this.data.length) {
+      throw new Error('Unexpected end of protobuf message')
+    }
+    return this.data[this.offset++]!
+  }
+
+  private skipBytes(length: number) {
+    const end = this.offset + length
+    if (Number.isSafeInteger(length) === false || end > this.data.length) {
+      throw new Error('Invalid protobuf message length')
+    }
+    this.offset = end
+  }
+
+  private expectWireType(actual: number, expected: number) {
+    if (actual !== expected) {
+      throw new Error(`Invalid protobuf wire type ${actual}, expected ${expected}`)
+    }
+  }
+}