uniswap-v2-loader 5.0.9 → 5.0.11

package/README.md

@@ -18,16 +18,11 @@
 
 ## CLI
 ```bash
-npm i -g uniswap-v2-loader
-uniswap-v2-loader --help
-Options:
- 	--key
-	--factory
-	--filename
-	--multicall_size
-	--from
-	--to
-	--workers
+> npm i -g uniswap-v2-loader
+> uniswap-v2-loader --from=1 --to=3
+
+2,0x12ede161c702d1494612d19f05992f43aa6a26fb,0x06af07097c9eeb7fd685c692751d5c66db49c215,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
+3,0xa478c2975ab1ea89e8196811f51a7b7ade33eb11,0x6b175474e89094c44da98b954eedeac495271d0f,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
 ```
 
 ## API Reference

package/bin/uniswap-v2-loader

@@ -2,15 +2,6 @@
 const { load } = require('../index')
 const rl = require('readline')
 
-const progress = (c, t) => {
-	const total_len = t.toString().length
-	const cur = c.toString().padStart(total_len)
-	const pct = (c / t * 100 | 0).toString().padStart(3)
-	rl.cursorTo(process.stdout, 0)
-	rl.clearLine(process.stdout, 0)
-	process.stdout.write(`Loaded: ${cur} / ${t} (${pct}%)`)
-}
-
 const options = [
     'key',
     'factory',
@@ -21,7 +12,7 @@ const options = [
     'workers',
 ]
 
-const params = { progress }
+const params = {}
 var i = 2
 while (arg = process.argv[i]) {
     if (arg == '-h' || arg == '--help') {
@@ -45,8 +36,40 @@ while (arg = process.argv[i]) {
             process.exit(22)
         }
     }
+	if (option == 'from' || option == 'to') {
+		value = Number(value)
+		if (isNaN(value) || value < 0 || !Number.isInteger(value)) {
+			console.log(`"${option}" should be a numeric index of pair at factory.`)
+			process.exit(22)
+		}
+	}
+	if (option == 'workers') {
+		value = Number(value)
+		if (isNaN(value) || value < 0 || !Number.isInteger(value)) {
+			console.log(`"workers" should be a numeric and positive integer.`)
+			process.exit(22)
+		}
+	}
+	if (option == 'multicall_size') {
+		value = Number(value)
+		if (isNaN(value) || value < 1 || !Number.isInteger(value)) {
+			console.log(`"multicall_size" should be a numeric. Not less then 1 request.`)
+			process.exit(22)
+		}
+	}
     params[option] = value
     i++
 }
 
+params.progress = params.filename
+	? (c, t, pair) => {
+		const total_left = t.toString().length
+		const cur = c.toString().padStart(total_left)
+		const pct = String(Math.round(100 * c / t)).padStart(3)
+		rl.cursorTo(process.stdout, 0)
+		rl.clearLine(process.stdout, 0)
+		process.stdout.write(`Loaded: ${cur} / ${t} (${pct}%)`)
+	}
+	: (c, t, _) => console.log(`${_.id},${_.pair},${_.token0},${_.token1}`)
+
 load(params)

package/index.js

@@ -20,6 +20,7 @@ const load = (params = {}) => {
         pairs,
     } = params
     filename ??= default_cache_filename(factory)
+    workers = Math.min(workers, max_workers)
 
     pairs ??= fs.existsSync(filename)
         ? fs.readFileSync(filename).toString().trim().split('\n')
@@ -37,8 +38,14 @@ const load = (params = {}) => {
             }, [])
         : []
 
-    if (to && pairs.length >= to) return Promise.resolve(pairs.slice(0, to))
+    if (to >= 0 && pairs.length >= to) {
+        if (progress)
+            for (var i = from; i < to; i++)
+                progress(pairs[i].id, to, pairs[i])
 
+        return Promise.resolve(pairs.slice(0, to))
+    }
+    
     return (to
         ? Promise.resolve(to)
         : fetch('https://eth-mainnet.g.alchemy.com/v2/' + key, {
@@ -50,27 +57,35 @@ const load = (params = {}) => {
                 method: 'eth_call',
                 params: [{ to: factory, data: '0x574f2ba3' }, 'latest']
             })
-        }).then(_ => _.json()).then(_ => Number(_.result))
+        }).then(
+            _ => {
+                if (_.ok) return _.json().then(_ => Number(_.result))
+                throw 'fail start'
+            },
+            _ => {
+                throw 'fetch failed'
+            }
+        )
     ).then(all_pairs_length => {
         const start_loading_from = pairs.length
-            ? Math.max(from || 0, pairs[pairs.length - 1].id + 1)
+            ? Math.max(from, pairs[pairs.length - 1].id + 1)
             : 0
 
         var next_pair_order = pairs.length
             ? pairs[pairs.length - 1].id + 1
             : 0
-        var progress_i = 0
-        const progress_end = all_pairs_length - start_loading_from
+
+        if (progress)
+            for (var i = from; i < start_loading_from; i++)
+                progress(pairs[i].id, all_pairs_length, pairs[i])
         
         const onpair = pair => {
             pairs[pair.id] = pair
-            if (progress) progress(++progress_i, progress_end)
-            if (filename) {
-                var _
-                while (_ = pairs[next_pair_order]) {
-                    fs.appendFileSync(filename, `${_.id},${_.pair},${_.token0},${_.token1}\n`)
-                    next_pair_order++
-                }
+            if (progress) progress(pair.id, all_pairs_length, pair)
+            var _
+            while (_ = pairs[next_pair_order]) {
+                fs.appendFileSync(filename, `${_.id},${_.pair},${_.token0},${_.token1}\n`)
+                next_pair_order++
             }
         }
 
@@ -80,7 +95,6 @@ const load = (params = {}) => {
                 ids.push(i)
             return require('./loader')({ ids, factory, key, multicall_size }, onpair)
             .then(() => pairs)
-        
         }
 
         const missed = []
@@ -105,6 +119,7 @@ const load = (params = {}) => {
             }))
         ).then(() => pairs)
     })
+    .catch(() => new Promise(resolve => setTimeout(() => resolve(load(params)), 1000)))
 }
 
 module.exports.load = (params = {}) =>

package/loader.js

@@ -9,7 +9,16 @@ const get_pairs_addresses = (key, factory, ids) => ids.length == 0
             method: 'eth_call',
             params: [{ to: factory, data: '0x1e3dd18b' + id.toString(16).padStart(64, '0') }, 'latest']
         })))
-    }).then(_ => _.json()).then(responds => {
+    }).then(
+        _ => {
+            if (_.ok) return _.json()
+            throw 'respond not ok'
+        },
+        _ => {
+            throw 'failed fetch'
+        }
+    )    
+    .then(responds => {
         responds.sort((a, b) => a.id - b.id)
         const addresses = []
         const failed_ids = []
@@ -22,6 +31,7 @@ const get_pairs_addresses = (key, factory, ids) => ids.length == 0
             ? addresses
             : get_pairs_addresses(key, factory, failed_ids).then(retried => [...addresses, ...retried])
     })
+    .catch(() => new Promise(resolve => setTimeout(() => resolve(get_pairs_addresses(key, factory, ids)), 1000)))
 
 const get_tokens = (key, addresses) => addresses.length == 0
     ? Promise.resolve({})
@@ -42,7 +52,15 @@ const get_tokens = (key, addresses) => addresses.length == 0
                 params: [{ to: address, data: '0xd21220a7' }, 'latest']
             }
         ]))
-    }).then(_ => _.json()).then(responds => {
+    }).then(
+        _ => {
+            if (_.ok) return _.json()
+            throw 'respond not ok'
+        },
+        _ => {
+            throw 'fetch failed'
+        }
+    ).then(responds => {
         responds.sort((a, b) => a.id - b.id)
         const tokens = {}
         const failed_addresses = []
@@ -64,6 +82,7 @@ const get_tokens = (key, addresses) => addresses.length == 0
             ? tokens
             : get_tokens(key, failed_addresses).then(retried => ({ ...tokens, ...retried }))
     })
+    .catch(() => new Promise(resolve => setTimeout(() => resolve(get_tokens(key, addresses)), 1000)))
 
 const main = ({ids, factory, key, multicall_size}, onpair) => {
     const chunks = []

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "uniswap-v2-loader",
-  "version": "5.0.9",
+  "version": "5.0.11",
   "description": "Uniswap v2 protocol loader",
   "keywords": [
     "uniswap-v2",

package/10.csv

@@ -1,10 +0,0 @@
-0,0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-1,0x3139ffc91b99aa94da8a2dc13f1fc36f9bdc98ee,0x8e870d67f660d95d5be530380d0ec0bd388289e1,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-2,0x12ede161c702d1494612d19f05992f43aa6a26fb,0x06af07097c9eeb7fd685c692751d5c66db49c215,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-3,0xa478c2975ab1ea89e8196811f51a7b7ade33eb11,0x6b175474e89094c44da98b954eedeac495271d0f,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-4,0x07f068ca326a469fc1d87d85d448990c8cba7df9,0x408e41876cccdc0f92210600ef50372656052a38,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-5,0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5,0x6b175474e89094c44da98b954eedeac495271d0f,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-6,0xce407cd7b95b39d3b4d53065e711e713dd5c5999,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0xfa3e941d1f6b7b10ed84a0c211bfa8aee907965e
-7,0x33c2d48bc95fb7d0199c5c693e7a9f527145a9af,0x1f573d6fb3f13d689ff844b4ce37794d79a7ff1c,0x6b175474e89094c44da98b954eedeac495271d0f
-8,0xb6909b960dbbe7392d405429eb2b3649752b4838,0x0d8775f648430679a709e98d2b0cb6250d2887ef,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-9,0x30eb5e15476e6a80f4f3cd8479749b4881dab1b8,0x39aa39c021dfbae8fac545936693ac917d5e7563,0x5d3a536e4d6dbd6114cc1ead35777bab948e3643

package/7.csv

@@ -1,7 +0,0 @@
-0,0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-1,0x3139ffc91b99aa94da8a2dc13f1fc36f9bdc98ee,0x8e870d67f660d95d5be530380d0ec0bd388289e1,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-2,0x12ede161c702d1494612d19f05992f43aa6a26fb,0x06af07097c9eeb7fd685c692751d5c66db49c215,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-3,0xa478c2975ab1ea89e8196811f51a7b7ade33eb11,0x6b175474e89094c44da98b954eedeac495271d0f,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-4,0x07f068ca326a469fc1d87d85d448990c8cba7df9,0x408e41876cccdc0f92210600ef50372656052a38,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-5,0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5,0x6b175474e89094c44da98b954eedeac495271d0f,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-6,0xce407cd7b95b39d3b4d53065e711e713dd5c5999,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0xfa3e941d1f6b7b10ed84a0c211bfa8aee907965e

package/8.csv

@@ -1,8 +0,0 @@
-0,0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-1,0x3139ffc91b99aa94da8a2dc13f1fc36f9bdc98ee,0x8e870d67f660d95d5be530380d0ec0bd388289e1,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-2,0x12ede161c702d1494612d19f05992f43aa6a26fb,0x06af07097c9eeb7fd685c692751d5c66db49c215,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-3,0xa478c2975ab1ea89e8196811f51a7b7ade33eb11,0x6b175474e89094c44da98b954eedeac495271d0f,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-4,0x07f068ca326a469fc1d87d85d448990c8cba7df9,0x408e41876cccdc0f92210600ef50372656052a38,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-5,0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5,0x6b175474e89094c44da98b954eedeac495271d0f,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-6,0xce407cd7b95b39d3b4d53065e711e713dd5c5999,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0xfa3e941d1f6b7b10ed84a0c211bfa8aee907965e
-7,0x33c2d48bc95fb7d0199c5c693e7a9f527145a9af,0x1f573d6fb3f13d689ff844b4ce37794d79a7ff1c,0x6b175474e89094c44da98b954eedeac495271d0f

package/9.csv

@@ -1,9 +0,0 @@
-0,0xb4e16d0168e52d35cacd2c6185b44281ec28c9dc,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-1,0x3139ffc91b99aa94da8a2dc13f1fc36f9bdc98ee,0x8e870d67f660d95d5be530380d0ec0bd388289e1,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-2,0x12ede161c702d1494612d19f05992f43aa6a26fb,0x06af07097c9eeb7fd685c692751d5c66db49c215,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-3,0xa478c2975ab1ea89e8196811f51a7b7ade33eb11,0x6b175474e89094c44da98b954eedeac495271d0f,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2
-4,0x07f068ca326a469fc1d87d85d448990c8cba7df9,0x408e41876cccdc0f92210600ef50372656052a38,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-5,0xae461ca67b15dc8dc81ce7615e0320da1a9ab8d5,0x6b175474e89094c44da98b954eedeac495271d0f,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48
-6,0xce407cd7b95b39d3b4d53065e711e713dd5c5999,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0xfa3e941d1f6b7b10ed84a0c211bfa8aee907965e
-7,0x33c2d48bc95fb7d0199c5c693e7a9f527145a9af,0x1f573d6fb3f13d689ff844b4ce37794d79a7ff1c,0x6b175474e89094c44da98b954eedeac495271d0f
-8,0xb6909b960dbbe7392d405429eb2b3649752b4838,0x0d8775f648430679a709e98d2b0cb6250d2887ef,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2