uniswap-v2-loader 6.0.0 → 6.1.0

package/README.md

@@ -68,9 +68,16 @@ Methods:
   * Type: `number`
   * Default: `undefined`
 - `filename`
-  * CSV cache path.
+  * Cache path. Used for pre-load data and add updates. In case `csv` set to `false` value used as prefix for:
+    - `${filename}_pairs.bin`
+    - `${filename}_tokens.bin`
+    - `${filename}_p2tt.bin`
   * Type: `string`
   * Default: *OS cache folder*
+- `csv`
+  * Switch cache between CSV and binary via [DEX DB](https://github.com/calp-pro/dex_db)
+  * Type: `boolean`
+  * Default: `true`
 - `factory`
   * Smart contract factory address.
   * Type: `string`
@@ -96,13 +103,13 @@ Methods:
   * Type: `function`
   * Default: `undefined`
 - `abort_signal`
-* Signal to cancel loading and release workers.
-* Type: `AbortSignal`
-* Default: `undefined`
+  * Signal to cancel loading and release workers.
+  * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal)
+  * Default: `undefined`
 - `update_timeout`
-* Polling interval in milliseconds. Used only in `subscribe`
-* Type: `number`
-* Default: `5000`
+  * Polling interval in milliseconds. Used only in `subscribe`
+  * Type: `number`
+  * Default: `5000`
 
 ### Schema `Pair`
 Standardized liquidity pool object.
@@ -133,7 +140,15 @@ The loader automatically identifies the optimal persistent storage path for your
 - **macOS:** `~/Library/Caches/`
 - **Windows:** `%LOCALAPPDATA%` or `AppData/Local/`
 
-Cache files are named following the pattern `${package_name}_{factory_address}.csv`.
+There are binary compact cache an CSV.<br>
+Binary cache done via [DEX DB](https://github.com/calp-pro/dex_db)
+
+Cache files are named following the pattern:
+- `${package_name}_{factory_address}.csv`
+- `${package_name}_{factory_address}_pairs.bin`
+- `${package_name}_{factory_address}_tokens.bin`
+- `${package_name}_{factory_address}_p2tt.bin`
+
 
 
 ## API Usage

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "uniswap-v2-loader",
-  "version": "6.0.0",
+  "version": "6.1.0",
   "description": "Uniswap v2 protocol loader",
   "keywords": [
     "DEFI",

package/src/default_cache_filename.js

@@ -19,5 +19,5 @@ module.exports = factory => path.join(
                         ? [home, '.cache']
                         : [os.tmpdir()]
     ),
-    `${pkg.name}_${factory.toLowerCase()}.csv`
+    `${pkg.name}_${factory.toLowerCase()}`
 )

package/src/dex_db.js

@@ -0,0 +1,216 @@
+// DexDB v2.0.0
+// written 13 March 2026 by Vladimir Spirin at Danang, Vietnam
+const fs = require('fs')
+
+function writeUInt24LE(buf, value, offset) {
+    value &= 0xffffff
+    buf[offset] = value & 0xff
+    buf[offset + 1] = (value >>> 8) & 0xff
+    buf[offset + 2] = (value >>> 16) & 0xff
+}
+
+function readUInt24LE(buf, offset) {
+    return buf[offset] |
+           (buf[offset + 1] << 8) |
+           (buf[offset + 2] << 16)
+}
+
+function dex_db(pairs = []) {
+    var aP = []
+    var aT = []
+    var P = new Map()//pairs
+    var T = new Map()//tokens
+    var p2tt = []// [p1.it0, p1.it1, p2.it0, p2, it1, ...]
+    var t2pt = []// [ [[ip, it], [ip, it], ...], ...]
+
+    const index = ([pair, token0, token1]) => {
+        var ip = P.get(pair)
+        if (ip == undefined) {
+            P.set(pair, ip = P.size)
+            aP.push(pair)
+        } else {
+            return undefined
+        }
+
+        var it0 = T.get(token0)
+        if (it0 == undefined) {
+            T.set(token0, it0 = T.size)
+            aT.push(token0)
+        }
+
+        var it1 = T.get(token1)
+        if (it1 == undefined) {
+            T.set(token1, it1 = T.size)
+            aT.push(token1)
+        }
+
+        p2tt[ip * 2] = it0
+        p2tt[ip * 2 + 1] = it1
+        
+        if (t2pt[it0])
+            t2pt[it0].push(ip, it1)
+        else
+            t2pt[it0] = [ip, it1]
+            
+        if (t2pt[it1])
+            t2pt[it1].push(ip, it0)
+        else
+            t2pt[it1] = [ip, it0]
+    }
+
+    const index_save = ([pair, token0, token1], filename = 'dump') => {
+        if (pair.length != 42 || token0.length != 42 || token1.length != 42) return
+        var ip = P.get(pair)
+        if (ip == undefined) {
+            P.set(pair, ip = P.size)
+            aP.push(pair)
+            fs.appendFileSync(filename + '_pairs.bin', Buffer.from(pair.slice(2), 'hex'))
+        } else {
+            return undefined
+        }
+
+        var it0 = T.get(token0)
+        if (it0 == undefined) {
+            T.set(token0, it0 = T.size)
+            aT.push(token0)
+            fs.appendFileSync(filename + '_tokens.bin', Buffer.from(token0.slice(2), 'hex'))
+        }
+
+        var it1 = T.get(token1)
+        if (it1 == undefined) {
+            T.set(token1, it1 = T.size)
+            aT.push(token1)
+            fs.appendFileSync(filename + '_tokens.bin', Buffer.from(token1.slice(2), 'hex'))
+        }
+
+        p2tt[ip * 2] = it0
+        p2tt[ip * 2 + 1] = it1
+
+        const buf = Buffer.allocUnsafe(6)
+        writeUInt24LE(buf, it0, 0)
+        writeUInt24LE(buf, it1, 3)
+        fs.appendFileSync(filename + '_p2tt.bin', buf)        
+        
+        if (t2pt[it0])
+            t2pt[it0].push(ip, it1)
+        else
+            t2pt[it0] = [ip, it1]
+            
+        if (t2pt[it1])
+            t2pt[it1].push(ip, it0)
+        else
+            t2pt[it1] = [ip, it0]
+
+        return [ip, it0, it1]
+    }
+
+    const get_tokens = pair => {
+        const tokens = Array(2)
+        const ip = P.get(pair)
+        if (ip == undefined) return tokens
+        tokens[0] = aT[p2tt[ip * 2]]
+        tokens[1] = aT[p2tt[ip * 2 + 1]]
+        return tokens
+    }
+
+    const find_pairs_with_token = token => {
+        const pairs = []
+        const it = T.get(token)
+        if (it == undefined) return pairs
+        for (var i = 0; i < t2pt[it].length; i += 2)
+            pairs.push(
+                aP[t2pt[it][i]]
+            )
+        return pairs
+    }
+
+    const find_pairs_with_tokens = (token0, token1) => {
+        const pairs = []
+        const it0 = T.get(token0)
+        if (it0 == undefined) return pairs
+        const it1 = T.get(token1)
+        if (it1 == undefined) return pairs
+        for (var i = 0; i < t2pt[it0].length; i += 2)
+            if (t2pt[it0][i + 1] == it1)
+                pairs.push(
+                    aP[t2pt[it0][i]]
+                )
+        return pairs
+    }
+    
+    const save = (filename = 'dump') => {
+        fs.writeFileSync(filename + '_pairs.bin', Buffer.concat(aP.map(a => Buffer.from(a.slice(2), 'hex'))))
+        fs.writeFileSync(filename + '_tokens.bin', Buffer.concat(aT.map(a => Buffer.from(a.slice(2), 'hex'))))
+        const bin = fs.openSync(filename + '_p2tt.bin', 'w')
+        const buf = Buffer.allocUnsafe(6)
+        for (var i = 0; i < p2tt.length; i += 2) {
+            writeUInt24LE(buf, p2tt[i], 0)
+            writeUInt24LE(buf, p2tt[i + 1], 3)
+            fs.writeSync(bin, buf)
+        }
+        fs.closeSync(bin)
+    }
+
+    const load = (filename = 'dump') => {
+        aP.length = 0
+        aT.length = 0
+        p2tt.length = 0
+        t2pt.length = 0
+        T = new Map()
+        P = new Map()
+
+        var buf = fs.readFileSync(filename + '_pairs.bin')
+        for (var i = 0; i < buf.length; i += 20) {
+            const pair = '0x' + buf.slice(i, i + 20).toString('hex')
+            aP.push(pair)
+            P.set(pair, i / 20)
+        }
+        buf = fs.readFileSync(filename + '_tokens.bin')
+        for (var i = 0; i < buf.length; i += 20) {
+            const token = '0x' + buf.slice(i, i + 20).toString('hex')
+            aT.push(token)
+            T.set(token, i / 20)
+        }
+
+        const bin = fs.openSync(filename + '_p2tt.bin', 'r')
+        const { size } = fs.fstatSync(bin)
+        buf = Buffer.allocUnsafe(6)
+        
+        for (var offset = 0, ip, it0, it1; offset < size; offset += 6) {
+            fs.readSync(bin, buf, 0, 6, offset)
+            ip = offset / 6
+            it0 = readUInt24LE(buf, 0)
+            it1 = readUInt24LE(buf, 3)
+            p2tt[ip * 2] = it0
+            p2tt[ip * 2 + 1] = it1
+            
+            if (t2pt[it0])
+                t2pt[it0].push(ip, it1)
+            else
+                t2pt[it0] = [ip, it1]
+                
+            if (t2pt[it1])
+                t2pt[it1].push(ip, it0)
+            else
+                t2pt[it1] = [ip, it0]
+        }
+
+        fs.closeSync(bin)
+    }
+    
+    pairs.forEach(index)
+    
+    return {
+        index,
+        index_save,
+        find_pairs_with_token,
+        find_pairs_with_tokens,
+        save,
+        load,
+        get_tokens,
+        get_all_pairs: () => aP,
+        get_all_tokens: () => aT,
+    }
+}
+
+module.exports = dex_db

package/src/index.js

@@ -3,6 +3,7 @@ const fs = require('fs')
 const os = require('os')
 const path = require('path')
 const default_cache_filename = require('./default_cache_filename')
+const dex_db = require('./dex_db')
 const max_workers = os.cpus().length - 1
 const debug_key = process.env.KEY || 'FZBvlPrOxtgaKBBkry3SH0W1IqH4Y5tu'
 const uniswap_v2_factory = '0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f'
@@ -12,6 +13,7 @@ const load = (params = {}) => {
         key = debug_key,
         factory = uniswap_v2_factory,
         filename,
+        csv = true,
         multicall_size = 50,
         from = 0,
         to,
@@ -21,23 +23,51 @@ const load = (params = {}) => {
         pairs,
     } = params
 
-    filename ??= default_cache_filename(factory)
+    if (!filename) {
+        filename = default_cache_filename(factory)
+        if (csv) filename += '.csv'
+    }
     workers = Math.min(workers, max_workers)
-
-    pairs ??= fs.existsSync(filename)
-        ? fs.readFileSync(filename).toString().trim().split('\n')
-            .reduce((pairs, line) => {
-                line = line.split(',')
-                const id = +line[0]
-                if (id >= from && (to == undefined || id <= to)) pairs.push({
-                    id,
-                    pair: line[1],
-                    token0: line[2],
-                    token1: line[3]
+    
+    var db
+    
+    if (!pairs) {
+        if (csv) {
+            pairs = fs.existsSync(filename)
+                ? fs.readFileSync(filename).toString().trim().split('\n')
+                    .reduce((pairs, line) => {
+                        line = line.split(',')
+                        const id = +line[0]
+                        if (id >= from && (to == undefined || id <= to)) pairs.push({
+                            id,
+                            pair: line[1],
+                            token0: line[2],
+                            token1: line[3]
+                        })
+                        return pairs
+                    }, [])
+                : []
+        } else {
+            pairs = []
+            db = dex_db()
+            if (
+                fs.existsSync(filename + '_pairs.bin') &&
+                fs.existsSync(filename + '_tokens.bin') &&
+                fs.existsSync(filename + '_p2tt.bin')
+            ) {
+                db.load(filename)
+                db.get_all_pairs().forEach((pair, i) => {
+                    const tokens = db.get_tokens(pair)
+                    pairs[i] = {
+                        id: i,
+                        pair,
+                        token0: tokens[0],
+                        token1: tokens[1]
+                    }
                 })
-                return pairs
-            }, [])
-        : []
+            }
+        }
+    }
 
     if (to >= 0 && to <= pairs.length - 1) {
         if (progress)
@@ -88,15 +118,25 @@ const load = (params = {}) => {
             for (var i = from; i < start_loading_from; i++)
                 progress(pairs[i].id, last_id + 1, pairs[i])
         
-        const onpair = pair => {
-            pairs[pair.id] = pair
-            if (progress && pair.id >= from) progress(pair.id, last_id + 1, pair)
-            var _
-            while (_ = pairs[next_pair_order]) {
-                fs.appendFileSync(filename, `${_.id},${_.pair},${_.token0},${_.token1}\n`)
-                next_pair_order++
+        const onpair = csv
+            ? pair => {
+                pairs[pair.id] = pair
+                if (progress && pair.id >= from) progress(pair.id, last_id + 1, pair)
+                var _
+                while (_ = pairs[next_pair_order]) {
+                    fs.appendFileSync(filename, `${_.id},${_.pair},${_.token0},${_.token1}\n`)
+                    next_pair_order++
+                }
+            }
+            : pair => {
+                pairs[pair.id] = pair
+                if (progress && pair.id >= from) progress(pair.id, last_id + 1, pair)
+                var _
+                while (_ = pairs[next_pair_order]) {
+                    db.index_save([_.pair, _.token0, _.token1], filename)
+                    next_pair_order++
+                }
             }
-        }
 
         if (!workers) {
             const ids = []

package/src/uniswap-v2-loader

@@ -6,6 +6,7 @@ const options = [
     'key',
     'factory',
     'filename',
+    'csv',
     'multicall_size',
     'from',
     'to',
@@ -57,10 +58,28 @@ while (arg = process.argv[i]) {
 			process.exit(22)
 		}
 	}
+    if (option == 'csv') {
+        if (value != 'true' && value != 'false') {
+            console.log(`"csv" should be a "true" or "false". Default is "true".`)
+			process.exit(22)
+		}
+        value = value == 'true'
+	}
     params[option] = value
     i++
 }
 
+if (params.csv != undefined && params.filename) {
+    if (params.csv == true && !params.filename.endsWith('.csv')) {
+        console.log(`CSV cache mode require "filename" ending with ".csv"`)
+        process.exit(22)
+    }
+    if (params.csv == false && params.filename.endsWith('.csv')) {
+        console.log(`Binary cache mode require "filename" ending without ".csv"`)
+        process.exit(22)
+    }
+}
+
 params.progress = params.filename
 	? (c, t, pair) => {
 		const total_left = t.toString().length