tardis-machine 18.6.2 → 18.8.0

package/node_modules/tardis-dev/dist/zstd.js

@@ -1,122 +1,8 @@
-import { once } from 'node:events';
-import { open } from 'node:fs/promises';
-import { Readable } from 'node:stream';
+import { createReadStream } from 'node:fs';
 import { createZstdDecompress } from 'node:zlib';
-const ZSTD_FRAME_MAGIC = 0xfd2fb528;
-const DICTIONARY_ID_SIZES = [0, 1, 2, 4];
 export function createZstdDecompressionStream(path, chunkSize) {
-    return Readable.from(decompressZstdFrames(path, chunkSize), { objectMode: false });
-}
-async function* decompressZstdFrames(path, chunkSize) {
-    const file = await open(path, 'r');
-    try {
-        const fileSize = (await file.stat()).size;
-        const frames = await findZstdFrames(file, path, fileSize);
-        for (const frame of frames) {
-            const source = Readable.from(readFileRange(file, frame.offset, frame.length, chunkSize), { objectMode: false });
-            const decompressor = createZstdDecompress({ chunkSize });
-            source.on('error', (error) => decompressor.destroy(error));
-            source.pipe(decompressor);
-            try {
-                for await (const chunk of decompressor) {
-                    yield chunk;
-                }
-                if (decompressor.bytesWritten !== frame.length) {
-                    throw new Error(`Zstd decoder consumed ${decompressor.bytesWritten} of ${frame.length} bytes at offset ${frame.offset} in ${path}`);
-                }
-            }
-            finally {
-                source.unpipe(decompressor);
-                source.destroy();
-                decompressor.destroy();
-                if (!source.closed) {
-                    await once(source, 'close');
-                }
-            }
-        }
-    }
-    finally {
-        await file.close();
-    }
-}
-async function findZstdFrames(file, path, fileSize) {
-    if (fileSize === 0) {
-        throw new Error(`Empty zstd file: ${path}`);
-    }
-    const bytes = Buffer.allocUnsafe(4);
-    const frames = [];
-    let offset = 0;
-    while (offset < fileSize) {
-        const frameStart = offset;
-        await readExactly(file, bytes, 4, offset, fileSize, path);
-        if (bytes.readUInt32LE(0) !== ZSTD_FRAME_MAGIC) {
-            throw new Error(`Invalid zstd frame magic at offset ${offset} in ${path}`);
-        }
-        offset += 4;
-        await readExactly(file, bytes, 1, offset, fileSize, path);
-        const descriptor = bytes[0];
-        if ((descriptor & 0x08) !== 0) {
-            throw new Error(`Unsupported reserved zstd frame descriptor bit at offset ${offset} in ${path}`);
-        }
-        offset++;
-        const singleSegment = (descriptor & 0x20) !== 0;
-        const contentChecksum = (descriptor & 0x04) !== 0;
-        const dictionaryIdSize = DICTIONARY_ID_SIZES[descriptor & 0x03];
-        const contentSizeFlag = descriptor >>> 6;
-        const contentSizeSize = contentSizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << contentSizeFlag;
-        const windowDescriptorSize = singleSegment ? 0 : 1;
-        offset += windowDescriptorSize + dictionaryIdSize + contentSizeSize;
-        ensureAvailable(offset, 0, fileSize, path);
-        let lastBlock = false;
-        while (!lastBlock) {
-            await readExactly(file, bytes, 3, offset, fileSize, path);
-            const blockHeader = bytes.readUIntLE(0, 3);
-            offset += 3;
-            lastBlock = (blockHeader & 1) !== 0;
-            const blockType = (blockHeader >>> 1) & 3;
-            if (blockType === 3) {
-                throw new Error(`Invalid reserved zstd block type at offset ${offset - 3} in ${path}`);
-            }
-            const blockSize = blockHeader >>> 3;
-            // RLE blocks store one repeated byte; other block types store blockSize bytes.
-            offset += blockType === 1 ? 1 : blockSize;
-            ensureAvailable(offset, 0, fileSize, path);
-        }
-        if (contentChecksum) {
-            offset += 4;
-            ensureAvailable(offset, 0, fileSize, path);
-        }
-        frames.push({ offset: frameStart, length: offset - frameStart });
-    }
-    return frames;
-}
-async function* readFileRange(file, position, length, chunkSize) {
-    let bytesRemaining = length;
-    while (bytesRemaining > 0) {
-        const buffer = Buffer.allocUnsafe(Math.min(bytesRemaining, chunkSize));
-        const { bytesRead } = await file.read(buffer, 0, buffer.length, position);
-        if (bytesRead === 0) {
-            throw new Error(`Expected ${length} compressed bytes, read ${length - bytesRemaining}`);
-        }
-        yield bytesRead === buffer.length ? buffer : buffer.subarray(0, bytesRead);
-        position += bytesRead;
-        bytesRemaining -= bytesRead;
-    }
-}
-async function readExactly(file, buffer, length, position, fileSize, path) {
-    ensureAvailable(position, length, fileSize, path);
-    let bytesRead = 0;
-    while (bytesRead < length) {
-        const result = await file.read(buffer, bytesRead, length - bytesRead, position + bytesRead);
-        if (result.bytesRead === 0) {
-            throw new Error(`Truncated zstd frame at offset ${position + bytesRead} in ${path}`);
-        }
-        bytesRead += result.bytesRead;
-    }
-}
-function ensureAvailable(position, length, fileSize, path) {
-    if (position < 0 || length < 0 || position > fileSize - length) {
-        throw new Error(`Truncated zstd frame at offset ${position} in ${path}`);
-    }
+    // Node supports rejectGarbageAfterEnd for Zstd; @types/node does not declare it yet.
+    const decoderOptions = { chunkSize, rejectGarbageAfterEnd: true };
+    return createReadStream(path, { highWaterMark: chunkSize }).compose(createZstdDecompress(decoderOptions));
 }
 //# sourceMappingURL=zstd.js.map

package/node_modules/tardis-dev/dist/zstd.js.map

@@ -1 +1 @@
-{"version":3,"file":"zstd.js","sourceRoot":"","sources":["../src/zstd.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AAClC,OAAO,EAAE,IAAI,EAAmB,MAAM,kBAAkB,CAAA;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAEhD,MAAM,gBAAgB,GAAG,UAAU,CAAA;AACnC,MAAM,mBAAmB,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;AAExC,MAAM,UAAU,6BAA6B,CAAC,IAAY,EAAE,SAAiB;IAC3E,OAAO,QAAQ,CAAC,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAA;AACpF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,oBAAoB,CAAC,IAAY,EAAE,SAAiB;IAClE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAClC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAA;QACzC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAA;QAEzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAA;YAC/G,MAAM,YAAY,GAAG,oBAAoB,CAAC,EAAE,SAAS,EAAE,CAAC,CAAA;YACxD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;YAC1D,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;YAEzB,IAAI,CAAC;gBACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;oBACvC,MAAM,KAAK,CAAA;gBACb,CAAC;gBAED,IAAI,YAAY,CAAC,YAAY,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;oBAC/C,MAAM,IAAI,KAAK,CACb,yBAAyB,YAAY,CAAC,YAAY,OAAO,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,MAAM,OAAO,IAAI,EAAE,CACnH,CAAA;gBACH,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;gBAC3B,MAAM,CAAC,OAAO,EAAE,CAAA;gBAChB,YAAY,CAAC,OAAO,EAAE,CAAA;gBACtB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACnB,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAgB,EAAE,IAAY,EAAE,QAAgB;IAC5E,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAA;IAC7C,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAA;IACnC,MAAM,MAAM,GAAyC,EAAE,CAAA;IACvD,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,OAAO,MAAM,GAAG,QAAQ,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,MAAM,CAAA;QACzB,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAA;QACzD,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,gBAAgB,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,OAAO,IAAI,EAAE,CAAC,CAAA;QAC5E,CAAC;QACD,MAAM,IAAI,CAAC,CAAA;QAEX,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAA;QACzD,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QAC3B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,4DAA4D,MAAM,OAAO,IAAI,EAAE,CAAC,CAAA;QAClG,CAAC;QACD,MAAM,EAAE,CAAA;QAER,MAAM,aAAa,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,eAAe,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAA;QACjD,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,UAAU,GAAG,IAAI,CAAC,CAAA;QAC/D,MAAM,eAAe,GAAG,UAAU,KAAK,CAAC,CAAA;QACxC,MAAM,eAAe,GAAG,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,eAAe,CAAA;QAC9F,MAAM,oBAAoB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAClD,MAAM,IAAI,oBAAoB,GAAG,gBAAgB,GAAG,eAAe,CAAA;QACnE,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAA;QAE1C,IAAI,SAAS,GAAG,KAAK,CAAA;QACrB,OAAO,CAAC,SAAS,EAAE,CAAC;YAClB,MAAM,WAAW,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAA;YACzD,MAAM,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,CAAA;YACX,SAAS,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;YACnC,MAAM,SAAS,GAAG,CAAC,WAAW,KAAK,CAAC,CAAC,GAAG,CAAC,CAAA;YACzC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,8CAA8C,MAAM,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;YACxF,CAAC;YAED,MAAM,SAAS,GAAG,WAAW,KAAK,CAAC,CAAA;YACnC,+EAA+E;YAC/E,MAAM,IAAI,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;YACzC,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAA;QAC5C,CAAC;QAED,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,CAAA;YACX,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAA;QAC5C,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU,EAAE,CAAC,CAAA;IAClE,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,IAAgB,EAAE,QAAgB,EAAE,MAAc,EAAE,SAAiB;IACjG,IAAI,cAAc,GAAG,MAAM,CAAA;IAC3B,OAAO,cAAc,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC,CAAA;QACtE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;QACzE,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,YAAY,MAAM,2BAA2B,MAAM,GAAG,cAAc,EAAE,CAAC,CAAA;QACzF,CAAC;QACD,MAAM,SAAS,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;QAC1E,QAAQ,IAAI,SAAS,CAAA;QACrB,cAAc,IAAI,SAAS,CAAA;IAC7B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAgB,EAAE,MAAc,EAAE,MAAc,EAAE,QAAgB,EAAE,QAAgB,EAAE,IAAY;IAC3H,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAA;IACjD,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,OAAO,SAAS,GAAG,MAAM,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,GAAG,SAAS,CAAC,CAAA;QAC3F,IAAI,MAAM,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,kCAAkC,QAAQ,GAAG,SAAS,OAAO,IAAI,EAAE,CAAC,CAAA;QACtF,CAAC;QACD,SAAS,IAAI,MAAM,CAAC,SAAS,CAAA;IAC/B,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,QAAgB,EAAE,MAAc,EAAE,QAAgB,EAAE,IAAY;IACvF,IAAI,QAAQ,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,QAAQ,GAAG,QAAQ,GAAG,MAAM,EAAE,CAAC;QAC/D,MAAM,IAAI,KAAK,CAAC,kCAAkC,QAAQ,OAAO,IAAI,EAAE,CAAC,CAAA;IAC1E,CAAC;AACH,CAAC"}
+{"version":3,"file":"zstd.js","sourceRoot":"","sources":["../src/zstd.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAEhD,MAAM,UAAU,6BAA6B,CAAC,IAAY,EAAE,SAAiB;IAC3E,qFAAqF;IACrF,MAAM,cAAc,GAAG,EAAE,SAAS,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAA;IACjE,OAAO,gBAAgB,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,cAAc,CAAC,CAAC,CAAA;AAC3G,CAAC"}

package/node_modules/tardis-dev/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tardis-dev",
-  "version": "18.6.2",
+  "version": "18.8.0",
   "engines": {
     "node": "^24.22.1 || >=26.12.1"
   },

package/node_modules/tardis-dev/src/consts.ts

@@ -35,6 +35,7 @@ export const EXCHANGES = [
   'bybit-spread',
   'phemex',
   'delta',
+  'delta-india',
   'ftx-us',
   'binance-us',
   'gate-io-futures',
@@ -475,6 +476,18 @@ const DELTA_CHANNELS = [
   'spot_30mtwap_price'
 ] as const
 
+const DELTA_INDIA_CHANNELS = [
+  'trades',
+  'ob_updates',
+  'ob_l1',
+  'ticker',
+  'mark_price',
+  'funding_rate',
+  'spot_price',
+  'product_updates',
+  'system_status'
+] as const
+
 const GATE_IO_CHANNELS = ['trades', 'depth', 'ticker', 'book_ticker', 'order_book_update', 'obu'] as const
 const GATE_IO_FUTURES_CHANNELS = ['trades', 'order_book', 'tickers', 'book_ticker', 'obu', 'public_liquidates'] as const
 const MEXC_FUTURES_CHANNELS = [
@@ -654,6 +667,7 @@ export const EXCHANGE_CHANNELS_INFO = {
   coinflex: COINFLEX_CHANNELS,
   phemex: PHEMEX_CHANNELS,
   delta: DELTA_CHANNELS,
+  'delta-india': DELTA_INDIA_CHANNELS,
   'gate-io': GATE_IO_CHANNELS,
   'gate-io-futures': GATE_IO_FUTURES_CHANNELS,
   'mexc-futures': MEXC_FUTURES_CHANNELS,

package/node_modules/tardis-dev/src/downloaddatasets.ts

@@ -2,11 +2,11 @@ import { existsSync } from 'node:fs'
 import { debug } from './debug.ts'
 import { DatasetType } from './exchangedetails.ts'
 import { forEachConcurrent } from './foreachconcurrent.ts'
-import { addDays, doubleDigit, download, parseAsUTCDate, sequence } from './handy.ts'
+import { addDays, doubleDigit, downloadDatasetFile, parseAsUTCDate, RequestSlots, sequence } from './handy.ts'
 import { getOptions } from './options.ts'
 import { Exchange } from './types.ts'
 
-const CONCURRENCY_LIMIT = 20
+const DEFAULT_CONCURRENCY = 20
 const MILLISECONDS_IN_SINGLE_DAY = 24 * 60 * 60 * 1000
 const DEFAULT_DOWNLOAD_DIR = './datasets'
 const DATASET_DOWNLOAD_ATTEMPT_TIMEOUT_MS = 120_000
@@ -19,6 +19,14 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
   const format = downloadDatasetsOptions.format !== undefined ? downloadDatasetsOptions.format : 'csv'
   const getFilename = downloadDatasetsOptions.getFilename !== undefined ? downloadDatasetsOptions.getFilename : getFilenameDefault
   const skipIfExists = downloadDatasetsOptions.skipIfExists === undefined ? true : downloadDatasetsOptions.skipIfExists
+  const verifyMd5 = downloadDatasetsOptions.verifyMd5 === true
+  const concurrency = downloadDatasetsOptions.concurrency !== undefined ? downloadDatasetsOptions.concurrency : DEFAULT_CONCURRENCY
+
+  if (concurrency < 1) {
+    throw new Error('Invalid "concurrency" argument. Please provide value greater than 0.')
+  }
+
+  const requestSlots = new RequestSlots(concurrency)
 
   // in case someone provided 'api/exchange' symbol, transform it to symbol that is accepted by datasets API
   const datasetsSymbols = symbols.map((s) => s.replace(/\/|:/g, '-').toUpperCase())
@@ -42,9 +50,11 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
             dataType,
             format,
             getFilename,
+            verifyMd5,
             date: addDays(startDate, daysCountToFetch - 1)
           }),
-          skipIfExists
+          skipIfExists,
+          requestSlots
         )
       }
 
@@ -60,15 +70,17 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
           dataType,
           format,
           getFilename,
+          verifyMd5,
           date: startDate
         }),
-        skipIfExists
+        skipIfExists,
+        requestSlots
       )
 
-      // download the rest concurrently up to the CONCURRENCY_LIMIT
+      // download the rest concurrently up to the concurrency limit
       await forEachConcurrent(
         sequence(daysCountToFetch - 1, 1), // this will produce Iterable sequence from 1 to daysCountToFetch - 1 (as we already downloaded data for the first and last day)
-        CONCURRENCY_LIMIT,
+        concurrency,
         (offset) =>
           downloadDataSet(
             getDownloadOptions({
@@ -81,9 +93,11 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
               dataType,
               format,
               getFilename,
+              verifyMd5,
               date: addDays(startDate, offset)
             }),
-            skipIfExists
+            skipIfExists,
+            requestSlots
           )
       )
       const elapsedSeconds = (new Date().valueOf() - startTimestamp) / 1000
@@ -93,12 +107,12 @@ export async function downloadDatasets(downloadDatasetsOptions: DownloadDatasets
   }
 }
 
-async function downloadDataSet(downloadOptions: DownloadOptions, skipIfExists: boolean) {
+async function downloadDataSet(downloadOptions: DownloadOptions, skipIfExists: boolean, requestSlots: RequestSlots) {
   if (skipIfExists && existsSync(downloadOptions.downloadPath)) {
     debug('dataset %s already exists, skipping download', downloadOptions.downloadPath)
     return
   } else {
-    return await download(downloadOptions)
+    return await downloadDatasetFile({ ...downloadOptions, requestSlots })
   }
 }
 
@@ -112,7 +126,8 @@ function getDownloadOptions({
   symbol,
   format,
   downloadDir,
-  getFilename
+  getFilename,
+  verifyMd5
 }: {
   exchange: Exchange
   dataType: DatasetType
@@ -124,6 +139,7 @@ function getDownloadOptions({
   userAgent: string
   downloadDir: string
   getFilename: (options: GetFilenameOptions) => string
+  verifyMd5: boolean
 }): DownloadOptions {
   const year = date.getUTCFullYear()
   const month = doubleDigit(date.getUTCMonth() + 1)
@@ -145,11 +161,12 @@ function getDownloadOptions({
     downloadPath,
     userAgent,
     apiKey,
-    attemptTimeoutMS: DATASET_DOWNLOAD_ATTEMPT_TIMEOUT_MS
+    attemptTimeoutMS: DATASET_DOWNLOAD_ATTEMPT_TIMEOUT_MS,
+    verifyMd5
   }
 }
 
-type DownloadOptions = Parameters<typeof download>[0]
+type DownloadOptions = Omit<Parameters<typeof downloadDatasetFile>[0], 'requestSlots'>
 
 export function sanitizeForFilename(s: string) {
   return s.replace(/[:\\/?*<>|"]/g, '-')
@@ -201,4 +218,6 @@ type DownloadDatasetsOptions = {
   downloadDir?: string
   getFilename?: (options: GetFilenameOptions) => string
   skipIfExists?: boolean
+  verifyMd5?: boolean
+  concurrency?: number
 }

package/node_modules/tardis-dev/src/handy.ts

@@ -1,11 +1,12 @@
 import crypto, { createHash } from 'crypto'
-import { createWriteStream, mkdirSync, rmSync } from 'node:fs'
+import { createReadStream, createWriteStream, mkdirSync, rmSync } from 'node:fs'
 import { rename } from 'node:fs/promises'
 import { Agent as HttpAgent, request as httpRequest, type IncomingMessage } from 'node:http'
 import { Agent as HttpsAgent, request as httpsRequest, type RequestOptions } from 'node:https'
 import { pipeline } from 'node:stream/promises'
 import path from 'path'
 import { debug } from './debug.ts'
+import { forEachConcurrent } from './foreachconcurrent.ts'
 import { Mapper } from './mappers/index.ts'
 import { Disconnect, Exchange, Filter, FilterForExchange } from './types.ts'
 
@@ -500,7 +501,7 @@ async function openDownloadResponse(url: string, options: DownloadRequestOptions
   const statusCode = response.statusCode ?? 0
   const location = response.headers.location
   if (location === undefined || statusCode < 300 || statusCode >= 400) {
-    return response
+    return { response }
   }
 
   response.destroy()
@@ -512,10 +513,13 @@ async function openDownloadResponse(url: string, options: DownloadRequestOptions
     delete headers.authorization
   }
 
-  return sendHttpRequest(redirectUrl, {
-    ...options,
-    headers
-  })
+  return {
+    response: await sendHttpRequest(redirectUrl, {
+      ...options,
+      headers
+    }),
+    redirectUrl: redirectUrl.toString()
+  }
 }
 
 async function readResponseText(response: IncomingMessage) {
@@ -619,6 +623,12 @@ export async function download({
     }
   }
 
+  return withDownloadRetries(url, downloadPath, (attemptUrl) =>
+    _downloadFile(httpRequestOptions, attemptUrl, downloadPath, appendContentEncodingExtension)
+  )
+}
+
+async function withDownloadRetries<T>(url: string, downloadPath: string, attempt: (attemptUrl: string) => Promise<T>) {
   const MAX_ATTEMPTS = 30
   const MAX_RETRY_AFTER_MS = 300 * ONE_SEC_IN_MS
   let attempts = 0
@@ -628,11 +638,7 @@ export async function download({
     attempts++
     try {
       const addRetryAttempt = attempts - 1 > 0 && url.endsWith('gz')
-      if (addRetryAttempt) {
-        return await _downloadFile(httpRequestOptions, `${url}?retryAttempt=${attempts - 1}`, downloadPath, appendContentEncodingExtension)
-      } else {
-        return await _downloadFile(httpRequestOptions, url, downloadPath, appendContentEncodingExtension)
-      }
+      return await attempt(addRetryAttempt ? `${url}?retryAttempt=${attempts - 1}` : url)
     } catch (error) {
       const unsupportedDataFeedEncoding = error instanceof Error && error.message.startsWith('Unsupported data feed content encoding')
       const nonRetryableHttpError =
@@ -640,13 +646,20 @@ export async function download({
         ((error.status === 400 && error.message.includes('ISO 8601 format') === false) ||
           error.status === 401 ||
           error.status === 403 ||
-          error.status === 404)
+          error.status === 404 ||
+          error.status === 416)
       const tooManyRequests = error instanceof HttpError && error.status === 429
       const retryAfterMS = tooManyRequests ? error.retryAfterMS : undefined
       const retryAfterTooLong = retryAfterMS !== undefined && retryAfterMS > MAX_RETRY_AFTER_MS
       const internalServiceError = error instanceof HttpError && error.status === 500
       // do not retry when we've got bad or unauthorized request, too long Retry-After or enough attempts
-      if (unsupportedDataFeedEncoding || nonRetryableHttpError || retryAfterTooLong || attempts === MAX_ATTEMPTS) {
+      if (
+        unsupportedDataFeedEncoding ||
+        error instanceof DatasetMismatchError ||
+        nonRetryableHttpError ||
+        retryAfterTooLong ||
+        attempts === MAX_ATTEMPTS
+      ) {
         throw error
       }
 
@@ -703,7 +716,7 @@ async function _downloadFile(
 
   try {
     // based on https://github.com/nodejs/node/issues/28172 - only reliable way to consume response stream and avoiding all the 'gotchas'
-    const response = await openDownloadResponse(url, requestOptions)
+    const { response } = await openDownloadResponse(url, requestOptions)
     const { statusCode } = response
     const responseHeaders = parseNodeResponseHeaders(response.headers)
     if (statusCode !== 200) {
@@ -741,6 +754,240 @@ async function _downloadFile(
   }
 }
 
+const DATASET_RANGE_SIZE = 64 * 1024 * 1024
+// A dataset is rarely exported again while it downloads, so repeated mismatches mean another restart won't help.
+const MAX_DATASET_RESTARTS = 3
+
+// The downloaded ranges don't make up one dataset file, because it was exported again during the download or doesn't
+// match its announced MD5, so the download starts over.
+class DatasetMismatchError extends Error {}
+
+// One pool for all files and ranges of a download, so splitting files into ranges does not add connections.
+export class RequestSlots {
+  private active = 0
+  private readonly waiting: (() => void)[] = []
+
+  constructor(readonly limit: number) {}
+
+  async run<T>(request: () => Promise<T>) {
+    if (this.active < this.limit) {
+      this.active++
+    } else {
+      await new Promise<void>((resolve) => this.waiting.push(resolve))
+    }
+
+    try {
+      return await request()
+    } finally {
+      const next = this.waiting.shift()
+      if (next === undefined) {
+        this.active--
+      } else {
+        next()
+      }
+    }
+  }
+}
+
+// Downloads byte ranges in parallel into one temp file, and a failed range retries alone.
+export async function downloadDatasetFile({
+  apiKey,
+  downloadPath,
+  url,
+  userAgent,
+  attemptTimeoutMS,
+  verifyMd5,
+  requestSlots
+}: {
+  url: string
+  downloadPath: string
+  userAgent: string
+  apiKey: string
+  attemptTimeoutMS: number
+  verifyMd5: boolean
+  requestSlots: RequestSlots
+}) {
+  // The API key goes only to the API, never to the storage the API redirects to.
+  const storageRequestOptions = {
+    timeout: attemptTimeoutMS,
+    headers: {
+      'Accept-Encoding': 'gzip',
+      'User-Agent': userAgent
+    }
+  }
+  const apiRequestOptions = {
+    ...storageRequestOptions,
+    headers: { ...storageRequestOptions.headers, Authorization: apiKey ? `Bearer ${apiKey}` : '' }
+  }
+
+  mkdirSync(path.dirname(downloadPath), { recursive: true })
+  const tmpFilePath = `${downloadPath}${crypto.randomBytes(8).toString('hex')}.unconfirmed`
+  const cleanup = () => {
+    try {
+      rmSync(tmpFilePath, { force: true })
+    } catch {}
+  }
+  tmpFileCleanups.set(tmpFilePath, cleanup)
+
+  try {
+    for (let restarts = 0; ; restarts++) {
+      try {
+        const announcedMd5 = await downloadDatasetRanges(url, apiRequestOptions, storageRequestOptions, tmpFilePath, requestSlots)
+        if (verifyMd5) {
+          await checkDatasetMd5(tmpFilePath, announcedMd5, url)
+        }
+        break
+      } catch (error) {
+        if (!(error instanceof DatasetMismatchError) || restarts === MAX_DATASET_RESTARTS) {
+          throw error
+        }
+      }
+    }
+
+    await rename(tmpFilePath, downloadPath)
+  } finally {
+    tmpFileCleanups.delete(tmpFilePath)
+    cleanup()
+  }
+}
+
+async function downloadDatasetRanges(
+  url: string,
+  apiRequestOptions: DownloadRequestOptions,
+  storageRequestOptions: DownloadRequestOptions,
+  tmpFilePath: string,
+  requestSlots: RequestSlots
+) {
+  const first = await withDownloadRetries(url, tmpFilePath, (attemptUrl) =>
+    requestSlots.run(() => downloadDatasetRange(attemptUrl, apiRequestOptions, tmpFilePath, 0))
+  )
+  const datasetSize = first.datasetSize
+  if (datasetSize === undefined || datasetSize <= DATASET_RANGE_SIZE) {
+    return first.md5
+  }
+
+  // A low-cost account gets a storage redirect that stays valid for two hours, and every redirect counts the whole
+  // dataset toward its transfer limit, so the remaining ranges reuse it and only an expired one is renewed.
+  let storageUrl = first.redirectUrl
+  let renewal: ReturnType<typeof downloadDatasetRange> | undefined
+
+  const downloadRemainingRange = async (attemptUrl: string, start: number) => {
+    // Read the URL only once the range has a slot, so a range that waited long uses the newest URL.
+    const rangeUrl = storageUrl
+    if (rangeUrl === undefined) {
+      return downloadDatasetRange(attemptUrl, apiRequestOptions, tmpFilePath, start, datasetSize, first.md5)
+    }
+
+    try {
+      return await downloadDatasetRange(rangeUrl, storageRequestOptions, tmpFilePath, start, datasetSize, first.md5)
+    } catch (error) {
+      if (!(error instanceof HttpError && error.status === 403)) {
+        throw error
+      }
+      // The first range to find the URL expired downloads through the API again, like the first range, and the
+      // redirect it gets replaces the URL. The others wait for it and retry with the new URL, or fail this attempt
+      // with its error.
+      if (renewal === undefined && storageUrl === rangeUrl) {
+        renewal = downloadDatasetRange(attemptUrl, apiRequestOptions, tmpFilePath, start, datasetSize, first.md5)
+          .then((renewedRange) => {
+            storageUrl = renewedRange.redirectUrl
+            return renewedRange
+          })
+          .finally(() => {
+            renewal = undefined
+          })
+        return await renewal
+      }
+      await renewal
+      throw new Error('Dataset redirect expired')
+    }
+  }
+
+  const rangeStarts = [...sequence(Math.ceil(datasetSize / DATASET_RANGE_SIZE), 1)].map((index) => index * DATASET_RANGE_SIZE)
+  // Unlike Promise.all, forEachConcurrent rejects only after the ranges already started have finished, retries included,
+  // so a restart or cleanup never overlaps a range still writing.
+  await forEachConcurrent(rangeStarts, requestSlots.limit, (start) =>
+    withDownloadRetries(url, tmpFilePath, (attemptUrl) => requestSlots.run(() => downloadRemainingRange(attemptUrl, start)))
+  )
+
+  return first.md5
+}
+
+async function downloadDatasetRange(
+  url: string,
+  requestOptions: DownloadRequestOptions,
+  tmpFilePath: string,
+  start: number,
+  expectedDatasetSize?: number,
+  expectedMd5?: string
+) {
+  const end = start + DATASET_RANGE_SIZE - 1
+  const { response, redirectUrl } = await openDownloadResponse(url, {
+    ...requestOptions,
+    headers: { ...requestOptions.headers, Range: `bytes=${start}-${end}` }
+  })
+  const statusCode = response.statusCode ?? 0
+  // A later range that comes back whole or out of bounds belongs to a dataset exported again with another size.
+  if (start > 0 && (statusCode === 200 || statusCode === 416)) {
+    response.destroy()
+    throw new DatasetMismatchError(`Dataset changed during download: bytes=${start}-${end} returned ${statusCode}`)
+  }
+  // 200 means the storage ignored the range and sends the whole dataset.
+  if (statusCode !== 206 && statusCode !== 200) {
+    throw new HttpError(statusCode, await readResponseText(response), url, parseNodeResponseHeaders(response.headers))
+  }
+
+  let datasetSize: number | undefined
+  if (statusCode === 206) {
+    const contentRange = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(asSingleHeaderValue(response.headers['content-range']) ?? '')
+    datasetSize = Number(contentRange?.[3])
+    if (contentRange === null || Number(contentRange[1]) !== start || Number(contentRange[2]) !== Math.min(end, datasetSize - 1)) {
+      response.destroy()
+      throw new Error(`Unexpected Content-Range for bytes=${start}-${end}: ${response.headers['content-range']}`)
+    }
+    if (expectedDatasetSize !== undefined && datasetSize !== expectedDatasetSize) {
+      response.destroy()
+      throw new DatasetMismatchError(`Dataset size changed from ${expectedDatasetSize} to ${datasetSize} bytes during download`)
+    }
+  }
+  // A dataset exported again with the same size announces another MD5.
+  const md5 = getAnnouncedMd5(parseNodeResponseHeaders(response.headers))
+  if (expectedMd5 !== undefined && md5 !== undefined && md5 !== expectedMd5) {
+    response.destroy()
+    throw new DatasetMismatchError(`Dataset MD5 changed from ${expectedMd5} to ${md5} during download`)
+  }
+
+  await pipeline(response, createWriteStream(tmpFilePath, { flags: start === 0 ? 'w' : 'r+', start }))
+  if (!response.complete) {
+    throw new Error('The connection was terminated while the message was still being sent')
+  }
+
+  return { datasetSize, redirectUrl, md5 }
+}
+
+// Ranges are written in parallel, so the MD5 is computed from the finished file.
+async function checkDatasetMd5(tmpFilePath: string, announcedMd5: string | undefined, url: string) {
+  if (announcedMd5 === undefined) {
+    return
+  }
+
+  const md5 = createHash('md5')
+  for await (const chunk of createReadStream(tmpFilePath)) {
+    md5.update(chunk)
+  }
+  const actualMd5 = md5.digest('hex')
+  if (actualMd5 !== announcedMd5) {
+    throw new DatasetMismatchError(`MD5 mismatch for ${url}: expected ${announcedMd5}, got ${actualMd5}`)
+  }
+}
+
+// The API sends x-md5, while a redirect to the storage returns the storage's x-amz-meta-md5 and ETag.
+// A multipart ETag ("<hash>-<parts>") is not the file MD5, so such a file is saved without a check.
+function getAnnouncedMd5(headers: Record<string, string>) {
+  const md5 = (headers['x-md5'] ?? headers['x-amz-meta-md5'] ?? headers['etag'] ?? '').replace(/"/g, '').toLowerCase()
+  return /^[0-9a-f]{32}$/.test(md5) ? md5 : undefined
+}
+
 function asSingleHeaderValue(headerValue: string | string[] | undefined) {
   if (Array.isArray(headerValue)) {
     return headerValue[0]

package/node_modules/tardis-dev/src/instrumentinfo.ts

@@ -162,7 +162,7 @@ export interface InstrumentInfo {
   contractMultiplier?: number
   /** only for quanto instruments */
   quanto?: boolean
-  /**  only for quanto instruments as settlement currency is different base/quote currency */
+  /** currency in which the derivative contract settles, when provided */
   settlementCurrency?: string
   /** strike price, only for options */
   strikePrice?: number