tardis-dev 12.5.22 → 12.6.7

package/src/downloaddatasets.ts

@@ -1,188 +1,188 @@
-import { existsSync } from 'fs-extra'
-import pMap from 'p-map'
-import { debug } from './debug'
-import { DatasetType } from './exchangedetails'
-import { addDays, doubleDigit, download, parseAsUTCDate, sequence } from './handy'
-import { getOptions } from './options'
-import { Exchange } from './types'
-
-const CONCURRENCY_LIMIT = 20
-const MILLISECONDS_IN_SINGLE_DAY = 24 * 60 * 60 * 1000
-const DEFAULT_DOWNLOAD_DIR = './datasets'
-
-const options = getOptions()
-
-export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasetsOptions) {
-  const { exchange, dataTypes, from, to, symbols } = downloadDatasetsOptions
-  const apiKey = downloadDatasetsOptions.apiKey !== undefined ? downloadDatasetsOptions.apiKey : options.apiKey
-  const downloadDir = downloadDatasetsOptions.downloadDir !== undefined ? downloadDatasetsOptions.downloadDir : DEFAULT_DOWNLOAD_DIR
-  const format = downloadDatasetsOptions.format !== undefined ? downloadDatasetsOptions.format : 'csv'
-  const getFilename = downloadDatasetsOptions.getFilename !== undefined ? downloadDatasetsOptions.getFilename : getFilenameDefault
-  const skipIfExists = downloadDatasetsOptions.skipIfExists === undefined ? true : downloadDatasetsOptions.skipIfExists
-
-  // in case someone provided 'api/exchange' symbol, transform it to symbol that is accepted by datasets API
-  const datasetsSymbols = symbols.map((s) => s.replace(/\/|:/g, '-').toUpperCase())
-
-  for (const symbol of datasetsSymbols) {
-    for (const dataType of dataTypes) {
-      const { daysCountToFetch, startDate } = getDownloadDateRange(downloadDatasetsOptions)
-      const startTimestamp = new Date().valueOf()
-      debug('dataset download started for %s %s %s from %s to %s', exchange, dataType, symbol, from, to)
-
-      if (daysCountToFetch > 1) {
-        // start with downloading last day of the range, validates is API key has access to the end range of requested data
-        await downloadDataSet(
-          getDownloadOptions({
-            exchange,
-            symbol,
-            apiKey,
-            downloadDir,
-            dataType,
-            format,
-            getFilename,
-            date: addDays(startDate, daysCountToFetch - 1)
-          }),
-          skipIfExists
-        )
-      }
-
-      // then download the first day of the range, validates is API key has access to the start range of requested data
-      await downloadDataSet(
-        getDownloadOptions({
-          exchange,
-          symbol,
-          apiKey,
-          downloadDir,
-          dataType,
-          format,
-          getFilename,
-          date: startDate
-        }),
-        skipIfExists
-      )
-
-      // download the rest concurrently up to the CONCURRENCY_LIMIT
-      await pMap(
-        sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
-        (offset) =>
-          downloadDataSet(
-            getDownloadOptions({
-              exchange,
-              symbol,
-              apiKey,
-              downloadDir,
-              dataType,
-              format,
-              getFilename,
-              date: addDays(startDate, offset)
-            }),
-            skipIfExists
-          ),
-        { concurrency: CONCURRENCY_LIMIT }
-      )
-      const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000
-
-      debug('dataset download finished for %s %s %s from %s to %s, time: %s seconds', exchange, dataType, symbol, from, to, elapsedSeconds)
-    }
-  }
-}
-
-async function downloadDataSet(downloadOptions: DownloadOptions, skipIfExists: boolean) {
-  if (skipIfExists && existsSync(downloadOptions.downloadPath)) {
-    debug('dataset %s already exists, skipping download', downloadOptions.downloadPath)
-  } else {
-    return await download(downloadOptions)
-  }
-}
-
-function getDownloadOptions({
-  apiKey,
-  exchange,
-  dataType,
-  date,
-  symbol,
-  format,
-  downloadDir,
-  getFilename
-}: {
-  exchange: Exchange
-  dataType: DatasetType
-  symbol: string
-  date: Date
-  format: string
-  apiKey: string
-  downloadDir: string
-  getFilename: (options: GetFilenameOptions) => string
-}): DownloadOptions {
-  const year = date.getUTCFullYear()
-  const month = doubleDigit(date.getUTCMonth() + 1)
-  const day = doubleDigit(date.getUTCDate())
-
-  const url = `${options.datasetsEndpoint}/${exchange}/${dataType}/${year}/${month}/${day}/${symbol}.${format}.gz`
-  const filename = getFilename({
-    dataType,
-    date,
-    exchange,
-    format,
-    symbol
-  })
-
-  const downloadPath = `${downloadDir}/${filename}`
-
-  return {
-    url,
-    downloadPath,
-    userAgent: options._userAgent,
-    apiKey
-  }
-}
-
-type DownloadOptions = Parameters<typeof download>[0]
-
-function getFilenameDefault({ exchange, dataType, format, date, symbol }: GetFilenameOptions) {
-  return `${exchange}_${dataType}_${date.toISOString().split('T')[0]}_${symbol}.${format}.gz`
-}
-
-function getDownloadDateRange({ from, to }: DownloadDatasetsOptions) {
-  if (!from || isNaN(Date.parse(from))) {
-    throw new Error(`Invalid "from" argument: ${from}. Please provide valid date string.`)
-  }
-
-  if (!to || isNaN(Date.parse(to))) {
-    throw new Error(`Invalid "to" argument: ${to}. Please provide valid date string.`)
-  }
-
-  const toDate = parseAsUTCDate(to)
-  const fromDate = parseAsUTCDate(from)
-  const daysCountToFetch = Math.floor((toDate.getTime() - fromDate.getTime()) / MILLISECONDS_IN_SINGLE_DAY)
-
-  if (daysCountToFetch < 1) {
-    throw new Error(`Invalid "to" and "from" arguments combination. Please provide "to" day that is later than "from" day.`)
-  }
-
-  return {
-    startDate: fromDate,
-    daysCountToFetch
-  }
-}
-
-type GetFilenameOptions = {
-  exchange: Exchange
-  dataType: DatasetType
-  symbol: string
-  date: Date
-  format: string
-}
-
-type DownloadDatasetsOptions = {
-  exchange: Exchange
-  dataTypes: DatasetType[]
-  symbols: string[]
-  from: string
-  to: string
-  format?: 'csv'
-  apiKey?: string
-  downloadDir?: string
-  getFilename?: (options: GetFilenameOptions) => string
-  skipIfExists?: boolean
-}
+import { existsSync } from 'fs-extra'
+import pMap from 'p-map'
+import { debug } from './debug'
+import { DatasetType } from './exchangedetails'
+import { addDays, doubleDigit, download, parseAsUTCDate, sequence } from './handy'
+import { getOptions } from './options'
+import { Exchange } from './types'
+
+const CONCURRENCY_LIMIT = 20
+const MILLISECONDS_IN_SINGLE_DAY = 24 * 60 * 60 * 1000
+const DEFAULT_DOWNLOAD_DIR = './datasets'
+
+const options = getOptions()
+
+export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasetsOptions) {
+  const { exchange, dataTypes, from, to, symbols } = downloadDatasetsOptions
+  const apiKey = downloadDatasetsOptions.apiKey !== undefined ? downloadDatasetsOptions.apiKey : options.apiKey
+  const downloadDir = downloadDatasetsOptions.downloadDir !== undefined ? downloadDatasetsOptions.downloadDir : DEFAULT_DOWNLOAD_DIR
+  const format = downloadDatasetsOptions.format !== undefined ? downloadDatasetsOptions.format : 'csv'
+  const getFilename = downloadDatasetsOptions.getFilename !== undefined ? downloadDatasetsOptions.getFilename : getFilenameDefault
+  const skipIfExists = downloadDatasetsOptions.skipIfExists === undefined ? true : downloadDatasetsOptions.skipIfExists
+
+  // in case someone provided 'api/exchange' symbol, transform it to symbol that is accepted by datasets API
+  const datasetsSymbols = symbols.map((s) => s.replace(/\/|:/g, '-').toUpperCase())
+
+  for (const symbol of datasetsSymbols) {
+    for (const dataType of dataTypes) {
+      const { daysCountToFetch, startDate } = getDownloadDateRange(downloadDatasetsOptions)
+      const startTimestamp = new Date().valueOf()
+      debug('dataset download started for %s %s %s from %s to %s', exchange, dataType, symbol, from, to)
+
+      if (daysCountToFetch > 1) {
+        // start with downloading last day of the range, validates is API key has access to the end range of requested data
+        await downloadDataSet(
+          getDownloadOptions({
+            exchange,
+            symbol,
+            apiKey,
+            downloadDir,
+            dataType,
+            format,
+            getFilename,
+            date: addDays(startDate, daysCountToFetch - 1)
+          }),
+          skipIfExists
+        )
+      }
+
+      // then download the first day of the range, validates is API key has access to the start range of requested data
+      await downloadDataSet(
+        getDownloadOptions({
+          exchange,
+          symbol,
+          apiKey,
+          downloadDir,
+          dataType,
+          format,
+          getFilename,
+          date: startDate
+        }),
+        skipIfExists
+      )
+
+      // download the rest concurrently up to the CONCURRENCY_LIMIT
+      await pMap(
+        sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
+        (offset) =>
+          downloadDataSet(
+            getDownloadOptions({
+              exchange,
+              symbol,
+              apiKey,
+              downloadDir,
+              dataType,
+              format,
+              getFilename,
+              date: addDays(startDate, offset)
+            }),
+            skipIfExists
+          ),
+        { concurrency: CONCURRENCY_LIMIT }
+      )
+      const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000
+
+      debug('dataset download finished for %s %s %s from %s to %s, time: %s seconds', exchange, dataType, symbol, from, to, elapsedSeconds)
+    }
+  }
+}
+
+async function downloadDataSet(downloadOptions: DownloadOptions, skipIfExists: boolean) {
+  if (skipIfExists && existsSync(downloadOptions.downloadPath)) {
+    debug('dataset %s already exists, skipping download', downloadOptions.downloadPath)
+  } else {
+    return await download(downloadOptions)
+  }
+}
+
+function getDownloadOptions({
+  apiKey,
+  exchange,
+  dataType,
+  date,
+  symbol,
+  format,
+  downloadDir,
+  getFilename
+}: {
+  exchange: Exchange
+  dataType: DatasetType
+  symbol: string
+  date: Date
+  format: string
+  apiKey: string
+  downloadDir: string
+  getFilename: (options: GetFilenameOptions) => string
+}): DownloadOptions {
+  const year = date.getUTCFullYear()
+  const month = doubleDigit(date.getUTCMonth() + 1)
+  const day = doubleDigit(date.getUTCDate())
+
+  const url = `${options.datasetsEndpoint}/${exchange}/${dataType}/${year}/${month}/${day}/${symbol}.${format}.gz`
+  const filename = getFilename({
+    dataType,
+    date,
+    exchange,
+    format,
+    symbol
+  })
+
+  const downloadPath = `${downloadDir}/${filename}`
+
+  return {
+    url,
+    downloadPath,
+    userAgent: options._userAgent,
+    apiKey
+  }
+}
+
+type DownloadOptions = Parameters<typeof download>[0]
+
+function getFilenameDefault({ exchange, dataType, format, date, symbol }: GetFilenameOptions) {
+  return `${exchange}_${dataType}_${date.toISOString().split('T')[0]}_${symbol}.${format}.gz`
+}
+
+function getDownloadDateRange({ from, to }: DownloadDatasetsOptions) {
+  if (!from || isNaN(Date.parse(from))) {
+    throw new Error(`Invalid "from" argument: ${from}. Please provide valid date string.`)
+  }
+
+  if (!to || isNaN(Date.parse(to))) {
+    throw new Error(`Invalid "to" argument: ${to}. Please provide valid date string.`)
+  }
+
+  const toDate = parseAsUTCDate(to)
+  const fromDate = parseAsUTCDate(from)
+  const daysCountToFetch = Math.floor((toDate.getTime() - fromDate.getTime()) / MILLISECONDS_IN_SINGLE_DAY)
+
+  if (daysCountToFetch < 1) {
+    throw new Error(`Invalid "to" and "from" arguments combination. Please provide "to" day that is later than "from" day.`)
+  }
+
+  return {
+    startDate: fromDate,
+    daysCountToFetch
+  }
+}
+
+type GetFilenameOptions = {
+  exchange: Exchange
+  dataType: DatasetType
+  symbol: string
+  date: Date
+  format: string
+}
+
+type DownloadDatasetsOptions = {
+  exchange: Exchange
+  dataTypes: DatasetType[]
+  symbols: string[]
+  from: string
+  to: string
+  format?: 'csv'
+  apiKey?: string
+  downloadDir?: string
+  getFilename?: (options: GetFilenameOptions) => string
+  skipIfExists?: boolean
+}

package/src/filter.ts

@@ -1,69 +1,69 @@
-import { NormalizedData, Disconnect, Trade } from './types'
-import { CappedSet } from './handy'
-
-export async function* filter<T extends NormalizedData | Disconnect>(messages: AsyncIterableIterator<T>, filter: (message: T) => boolean) {
-  for await (const message of messages) {
-    if (filter(message)) {
-      yield message
-    }
-  }
-}
-
-export function uniqueTradesOnly<T extends NormalizedData | Disconnect>(
-  {
-    maxWindow,
-    onDuplicateFound,
-    skipStaleOlderThanSeconds
-  }: {
-    maxWindow: number
-    skipStaleOlderThanSeconds?: number
-    onDuplicateFound?: (trade: Trade) => void
-  } = {
-    maxWindow: 500
-  }
-) {
-  const perSymbolQueues = {} as {
-    [key: string]: CappedSet<string>
-  }
-
-  return (message: T) => {
-    // pass trough any message that is not a trade
-    if (message.type !== 'trade') {
-      return true
-    } else {
-      const trade = (message as unknown) as Trade
-      // pass trough trades that can't be uniquely identified
-      // ignore index trades
-      if (trade.id === undefined || trade.symbol.startsWith('.')) {
-        return true
-      } else {
-        let alreadySeenTrades = perSymbolQueues[trade.symbol]
-
-        if (alreadySeenTrades === undefined) {
-          perSymbolQueues[trade.symbol] = new CappedSet<string>(maxWindow)
-          alreadySeenTrades = perSymbolQueues[trade.symbol]
-        }
-
-        const isDuplicate = alreadySeenTrades.has(trade.id)
-        const isStale =
-          skipStaleOlderThanSeconds !== undefined &&
-          trade.localTimestamp.valueOf() - trade.timestamp.valueOf() > skipStaleOlderThanSeconds * 1000
-
-        if (isDuplicate || isStale) {
-          if (onDuplicateFound !== undefined) {
-            onDuplicateFound(trade)
-          }
-          // refresh duplicated key position so it's added back at the beginning of the queue
-          alreadySeenTrades.remove(trade.id)
-          alreadySeenTrades.add(trade.id)
-
-          return false
-        } else {
-          alreadySeenTrades.add(trade.id)
-
-          return true
-        }
-      }
-    }
-  }
-}
+import { NormalizedData, Disconnect, Trade } from './types'
+import { CappedSet } from './handy'
+
+export async function* filter<T extends NormalizedData | Disconnect>(messages: AsyncIterableIterator<T>, filter: (message: T) => boolean) {
+  for await (const message of messages) {
+    if (filter(message)) {
+      yield message
+    }
+  }
+}
+
+export function uniqueTradesOnly<T extends NormalizedData | Disconnect>(
+  {
+    maxWindow,
+    onDuplicateFound,
+    skipStaleOlderThanSeconds
+  }: {
+    maxWindow: number
+    skipStaleOlderThanSeconds?: number
+    onDuplicateFound?: (trade: Trade) => void
+  } = {
+    maxWindow: 500
+  }
+) {
+  const perSymbolQueues = {} as {
+    [key: string]: CappedSet<string>
+  }
+
+  return (message: T) => {
+    // pass trough any message that is not a trade
+    if (message.type !== 'trade') {
+      return true
+    } else {
+      const trade = message as unknown as Trade
+      // pass trough trades that can't be uniquely identified
+      // ignore index trades
+      if (trade.id === undefined || trade.symbol.startsWith('.')) {
+        return true
+      } else {
+        let alreadySeenTrades = perSymbolQueues[trade.symbol]
+
+        if (alreadySeenTrades === undefined) {
+          perSymbolQueues[trade.symbol] = new CappedSet<string>(maxWindow)
+          alreadySeenTrades = perSymbolQueues[trade.symbol]
+        }
+
+        const isDuplicate = alreadySeenTrades.has(trade.id)
+        const isStale =
+          skipStaleOlderThanSeconds !== undefined &&
+          trade.localTimestamp.valueOf() - trade.timestamp.valueOf() > skipStaleOlderThanSeconds * 1000
+
+        if (isDuplicate || isStale) {
+          if (onDuplicateFound !== undefined) {
+            onDuplicateFound(trade)
+          }
+          // refresh duplicated key position so it's added back at the beginning of the queue
+          alreadySeenTrades.remove(trade.id)
+          alreadySeenTrades.add(trade.id)
+
+          return false
+        } else {
+          alreadySeenTrades.add(trade.id)
+
+          return true
+        }
+      }
+    }
+  }
+}

package/src/handy.ts

@@ -253,7 +253,7 @@ export async function download({
     }
   }
 
-  const MAX_ATTEMPTS = 8
+  const MAX_ATTEMPTS = 30
   let attempts = 0
 
   while (true) {
@@ -270,7 +270,7 @@ export async function download({
       }
 
       const randomIngridient = Math.random() * 500
-      const attemptsDelayMS = Math.pow(2, attempts) * ONE_SEC_IN_MS
+      const attemptsDelayMS = Math.min(Math.pow(2, attempts) * ONE_SEC_IN_MS, 120 * ONE_SEC_IN_MS)
       let nextAttemptDelayMS = randomIngridient + attemptsDelayMS
 
       if (tooManyRequests) {

package/src/instrumentinfo.ts

@@ -34,7 +34,7 @@ async function getInstrumentInfoForExchange(exchange: Exchange, filterOrSymbol?:
         headers: { Authorization: `Bearer ${options.apiKey}` }
       })
       .json()
-  } catch (e) {
+  } catch (e: any) {
     // expose 400 error message from server
     if (e.response?.statusCode === 400) {
       let err: { code: Number; message: string }