uniswap-v2-loader 1.5.0 → 2.0.0

package/bin/uniswap-v2-loader

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-const { all, count, clear_cache } = require('../index')
+const { load, count, clear_cache } = require('../index')
 const rl = require('readline')
 
 const progress = (c, t) => {
@@ -16,4 +16,4 @@ if (process.argv[2] == '-c' || process.argv[2] == '--count')
 else if (process.argv[2] == '--clear')
 	clear_cache()
 else 
-	all({progress})
+	load({progress})

package/index.js

@@ -4,19 +4,30 @@ const os = require('os')
 const path = require('path')
 const { parseAbiItem, createPublicClient, http } = require('viem')
 const { mainnet } = require('viem/chains')
-const default_filename = require('./default_cache_filename')
-const workers = os.cpus().length - 1
-const missed = Array(workers).fill(null).map(() => [])
-const key = process.env.KEY || 'FZBvlPrOxtgaKBBkry3SH0W1IqH4Y5tu'
-const factory = '0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f'
-const client = createPublicClient({
-    chain: mainnet,
-    transport: http('https://eth-mainnet.g.alchemy.com/v2/' + key)
-})
+const default_cache_filename = require('./default_cache_filename')
+const max_workers = os.cpus().length - 1
+const debug_key = process.env.KEY || 'FZBvlPrOxtgaKBBkry3SH0W1IqH4Y5tu'
+const uniswap_v2_factory = '0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f'
 
-const load = params => {
-    const {filename = default_filename, to, from = 0, multicall_size = 50, progress, count, multicore = true} = params
-    const pairs = params.pairs || fs.existsSync(filename)
+const load = (params = {}) => {
+    var {
+        key = debug_key,
+        factory = uniswap_v2_factory,
+        filename = default_cache_filename,
+        multicall_size = 50,
+        from = 0,
+        to,
+        progress,
+        count,
+        workers = max_workers,
+        pairs,
+    } = params
+    const client = createPublicClient({
+        chain: mainnet,
+        transport: http('https://eth-mainnet.g.alchemy.com/v2/' + key)
+    })
+
+    pairs ??= fs.existsSync(filename)
         ? fs.readFileSync(filename).toString().trim().split('\n')
             .reduce((pairs, line) => {
                 line = line.split(',')
@@ -52,8 +63,6 @@ const load = params => {
             : 0
         var progress_i = 0
         const progress_end = all_pairs_length - start_loading_from
-
-        missed.forEach(_ => _.length = 0)
         
         const onpair = pair => {
             pairs[pair.id] = pair
@@ -67,18 +76,23 @@ const load = params => {
             }
         }
 
-        if (workers == 0 || multicore == false) {
+        if (!workers) {
             const ids = []
             for (var i = start_loading_from; i < all_pairs_length; i++)
                 ids.push(i)
             return require('./loader')({ ids, factory, key, multicall_size }, onpair)
         
         }
-        for (var i = start_loading_from, rr = 0; i < all_pairs_length; i++) {
-            missed[rr].push(i)
-            if (missed[rr].length % multicall_size == 0)
-                rr = (rr + 1) % workers
+
+        const missed = []
+        for (var i = start_loading_from, ids; i < all_pairs_length; i++) {
+            if (!ids || ids.length % multicall_size == 0) {
+                ids = []
+                missed.push(ids)
+            }
+            ids.push(i)
         }
+        
         cluster.setupPrimary({ exec: path.join(__dirname, 'loader.js') })
         
         return Promise.all(
@@ -95,14 +109,16 @@ const load = params => {
 }
 
 
-module.exports.clear_cache = () =>
-    fs.unlinkSync(default_filename)
+module.exports.clear_cache = () => {
+    if (fs.existsSync(default_cache_filename))
+    fs.unlinkSync(default_cache_filename)
+}
 
-module.exports.all = (params = {}) =>
+module.exports.load = (params = {}) =>
     load(params)
     
-module.exports.count = () =>
-    load({count: true})
+module.exports.count = (params = {}) =>
+    load({count: true, ...params})
 
 module.exports.onupdate = function onupdate(callback, params = {}) {
     params.update_timeout ??= 5000
@@ -134,4 +150,4 @@ module.exports.onupdate = function onupdate(callback, params = {}) {
         subscribe = false
         if (timeout) clearTimeout(timeout)
     }
-}
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "uniswap-v2-loader",
-  "version": "1.5.0",
+  "version": "2.0.0",
   "description": "Uniswap v2 protocol loader",
   "keywords": [
     "uniswap-v2",
@@ -9,13 +9,13 @@
     "crypto",
     "ethereum"
   ],
-  "homepage": "https://github.com/calp-pro/uniswap-v2#readme",
+  "homepage": "https://github.com/calp-pro/uniswap-v2-loader#readme",
   "bugs": {
-    "url": "https://github.com/calp-pro/uniswap-v2/issues"
+    "url": "https://github.com/calp-pro/uniswap-v2-loader/issues"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/calp-pro/uniswap-v2.git"
+    "url": "git+https://github.com/calp-pro/uniswap-v2-loader.git"
   },
   "license": "MIT",
   "author": "Vladimir Spirin (spirin.vladimir@gmail.com)",

package/test.js

@@ -1,54 +1,67 @@
 const fs = require('fs')
-const { test } = require('node:test')
+const { describe, before, it } = require('node:test')
 const assert = require('node:assert/strict')
-const uniswap_v2_loader = require('./index')
+const {clear_cache, load, onupdate} = require('./index')
 
-test('Exist USDC/USDP pair', () =>
-    uniswap_v2_loader.all({to: 14})
-    .then(pairs => {
-        assert.equal(pairs.length, 14)
-        const i = pairs.findIndex(({id}) => id == 1)
-        assert.ok(i != -1)
-        if (i != -1) {
-            const {pair, token0, token1} = pairs[i]
-            assert.equal(pair, '0x3139ffc91b99aa94da8a2dc13f1fc36f9bdc98ee')
-            assert.equal(token0, '0x8e870d67f660d95d5be530380d0ec0bd388289e1')
-            assert.equal(token1, '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48')
-        }
+describe('Uniswap V2', () => {
+    before(() => clear_cache())
+    
+    it('Exist USDC/USDP pair', () =>
+        load({to: 2})
+        .then(pairs => {
+            assert.equal(pairs.length, 2)
+            const i = pairs.findIndex(({id}) => id == 1)
+            assert.ok(i != -1)
+            if (i != -1) {
+                const {pair, token0, token1} = pairs[i]
+                assert.equal(pair, '0x3139Ffc91B99aa94DA8A2dc13f1fC36F9BDc98eE')
+                assert.equal(token0, '0x8E870D67F660D95d5be530380D0eC0bd388289E1')
+                assert.equal(token1, '0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48')
+            }
+        })
+    )
+    
+    
+    it('Re-load first two pairs to custom CSV file', () => {
+        // If user specify a filename then
+        // a cache data will be taken from
+        // the filename provided. If file is empty then
+        // data will be uploaded again from network.
+        const filename = Date.now() + '.csv'
+        return load({to: 2, filename})
+        .then(() => {
+            const lines = fs.readFileSync(filename).toString().trim().split('\n')
+            assert.equal(lines.length, 2)
+        })
+        .finally(() =>
+            fs.unlinkSync(filename)
+        )
     })
-)
 
-test('Load first two pairs to file', () => {
-    const filename = Date.now() + '.csv'
-    return uniswap_v2_loader.all({to: 2, filename})
-    .then(() => {
-        const lines = fs.readFileSync(filename).toString().trim().split('\n')
-        assert.equal(lines.length, 2)
+    it('onupdate should call provided callback with 2 pairs for a current moment (from cache)', () => {
+        return new Promise(y => {
+            const unsubscribe = onupdate(pairs => {
+                assert.equal(pairs.length, 2)
+                unsubscribe()
+                y()
+            }, {to: 2})
+        })
     })
-    .finally(() =>
-        fs.unlinkSync(filename)
-    )
-})
 
-test('onupdate should call provided callback with 2 pairs for a current moment (from cache)', () => {
-    return new Promise(y => {
-        const unsubscribe = uniswap_v2_loader.onupdate(pairs => {
-            assert.equal(pairs.length, 2)
-            unsubscribe()
-            y()
-        }, {to: 2})
-    })
-})
+    it('Multi-core test 2 workers load 2 pools using multicall', () =>
+        // There are already 2 pools loaded from previous test
+        // 6 - 2 = 4. Rest 4 will be loaded by 2 workers. Each load 2.
+        // Multicall size is 2.
+        load({to: 6, multicall_size: 2, workers: 2 })
+        .then(pairs => {
+            assert.equal(pairs.length, 6)
+        })
+    )
 
-test('Heavy test load first 3000 pairs', () =>
-    uniswap_v2_loader.all({to: 3000})
-    .then(pairs => {
-        assert.equal(pairs.length, 3000)
+    it('Each line at CSV cache file should be orderd by pair id (factory id)', () => {
+        const lines = fs.readFileSync(require('./default_cache_filename'), 'utf8').trim().split('\n')
+        for (var i = 0; i < lines.length; i++)
+            assert.equal(i, +lines[i].split(',').shift())
     })
-)
 
-test('Each line at CSV cache file should be orderd by pair id (factory id)', () => {
-    const lines = fs.readFileSync(require('./default_cache_filename'), 'utf8').trim().split('\n')
-    for (var i = 0; i < lines.length; i++)
-        assert.equal(i, +lines[i].split(',').shift())
 })