uniswap-v2-loader 1.2.0 → 1.4.0

package/CHANGELOG.md

@@ -1,2 +1,10 @@
+## [1.4.0] - 2026-02-24
+- spawn -> cluster
+- test order pool by factory id at CSV
+- clear cache CSV file
+
+## [1.3.0] - 2026-02-23
+- Add CLI version with `-c` or `--count` flag to counting loaded pairs from cache
+
 ## [1.2.0] - 2026-02-22
 - Added `onupdate` function for subscribing to new pairs.

package/README.md

@@ -6,6 +6,13 @@ High-speed Uniswap v2 pair loader using viem multicall and parallel CPU processi
 The package uses Alchemy. Set your key as an environment variable (a default key is used if none is provided):
 `export KEY=your_alchemy_key`
 
+## CLI
+```
+npm i -g uniswap-v2-loader
+uniswap-v2-loader
+```
+
+
 ## API Reference
 ### `all(params)`
 - **Description**: Fetches token pairs from the Uniswap v2 factory. It utilizes multicall from `viem` and splits the loading process between multiple CPUs for high-speed execution.

package/bin/uniswap-v2-loader

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+const { all, count, clear_cache } = require('../index')
+const rl = require('readline')
+
+const progress = (c, t) => {
+	const total_len = t.toString().length
+	const cur = c.toString().padStart(total_len)
+	const pct = (c / t * 100 | 0).toString().padStart(3)
+	rl.cursorTo(process.stdout, 0)
+	rl.clearLine(process.stdout, 0)
+	process.stdout.write(`Loaded: ${cur} / ${t} (${pct}%)`)
+}
+
+if (process.argv[2] == '-c' || process.argv[2] == '--count')
+	console.log(count())
+else if (process.argv[2] == '--clear')
+	clear_cache()
+else 
+	all({progress})

package/debug.js

@@ -0,0 +1 @@
+require('./index').all({to: 487283})

package/default_cache_filename.js

@@ -0,0 +1,19 @@
+const path = require('path')
+const env = process.env
+const os = require('os')
+const home = os.homedir()
+const pkg = require('./package.json')
+
+module.exports = path.join(
+  ...(process.platform === 'win32'
+      ? (env.LOCALAPPDATA || env.APPDATA)
+        ? [env.LOCALAPPDATA || env.APPDATA]
+        : [home, 'AppData', 'Local']
+      : process.platform === 'darwin'
+        ? [home, 'Library', 'Caches']
+        : (env.XDG_CACHE_HOME && path.isAbsolute(env.XDG_CACHE_HOME))
+          ? [env.XDG_CACHE_HOME]
+          : [home, '.cache']
+  ),
+  pkg.name + '_pairs.csv'
+)

package/index.js

@@ -1,26 +1,9 @@
-const { spawn } = require('child_process')
+const cluster = require('cluster')
 const fs = require('fs')
 const os = require('os')
-const path = require('path')
-const env = process.env
-const home = os.homedir()
-const pkg = require('./package.json')
-const default_filename = path.join(
-  ...(process.platform === 'win32'
-      ? (env.LOCALAPPDATA || env.APPDATA)
-        ? [env.LOCALAPPDATA || env.APPDATA]
-        : [home, 'AppData', 'Local']
-      : process.platform === 'darwin'
-        ? [home, 'Library', 'Caches']
-        : (env.XDG_CACHE_HOME && path.isAbsolute(env.XDG_CACHE_HOME))
-          ? [env.XDG_CACHE_HOME]
-          : [home, '.cache']
-  ),
-  pkg.name + '_pairs.csv'
-)
 const { parseAbiItem, createPublicClient, http } = require('viem')
 const { mainnet } = require('viem/chains')
-
+const default_filename = require('./default_cache_filename')
 const workers = os.cpus().length - 1
 const missed = Array(workers).fill(null).map(() => [])
 const key = process.env.KEY || 'FZBvlPrOxtgaKBBkry3SH0W1IqH4Y5tu'
@@ -31,7 +14,7 @@ const client = createPublicClient({
 })
 
 const load = params => {
-    const {filename = default_filename, to, from = 0, chunk_size = 50} = params
+    const {filename = default_filename, to, from = 0, chunk_size = 50, progress, count} = params
     const pairs = params.pairs || fs.existsSync(filename)
         ? fs.readFileSync(filename).toString().trim().split('\n')
             .reduce((pairs, line) => {
@@ -47,7 +30,8 @@ const load = params => {
             }, [])
         : []
 
-    if (pairs.length > to) return Promise.resolve(pairs.slice(0, to))
+    if (count) return pairs.length
+    if (to && pairs.length > to) return Promise.resolve(pairs.slice(0, to))
 
     return (to
         ? Promise.resolve(to)
@@ -56,69 +40,60 @@ const load = params => {
             abi: [parseAbiItem('function allPairsLength() view returns (uint256)')],
             functionName: 'allPairsLength'
         }).then(_ => Number(_))
-    ).then(allPairsLength => {
-        var next_pair_order = 0    
+    ).then(all_pairs_length => {
         const start_loading_from = pairs.length
             ? Math.max(from || 0, pairs[pairs.length - 1].id + 1)
             : 0
 
+        var next_pair_order = pairs.length
+            ? pairs[pairs.length - 1].id + 1
+            : 0
+
         missed.forEach(_ => _.length = 0)
         
-        for (var i = start_loading_from, rr = 0; i < allPairsLength; i++) {
+        for (var i = start_loading_from, rr = 0; i < all_pairs_length; i++) {
             missed[rr].push(i)
             if (missed[rr].length % chunk_size == 0)
                 rr = (rr + 1) % workers
         }
         
-        const jobs_data_filename = `jobs_data_${Date.now()}.json`
-        fs.writeFileSync(jobs_data_filename, JSON.stringify({
-            missed,
-            factory,
-            chunk_size,
-            key
-        }), 'utf8')
+        var progress_i = 0
+        const progress_end = all_pairs_length - start_loading_from
         
+        cluster.setupPrimary({ exec: __dirname + '/loader.js' })
         return Promise.all(
             missed
             .filter(_ => _.length)
-            .map((_, i) => new Promise(y => {
-                const loader = spawn('node', ['loader.js', jobs_data_filename, i.toString()])
-                loader.stdout.on('data', data => {
-                    data += data.toString()
-                    if (!data.includes('\n')) return
-                    const lines = data.split('\n')
-                    data = lines.shift()
-                    lines.forEach(line => {
-                        const a = line.split(',')
-                        const id = +a[0]
-                        pairs[id] = {
-                            id,
-                            pair: a[1],
-                            token0: a[2],
-                            token1: a[3]
-                        }
-                    })
+            .map((missed, i) => new Promise(y => {
+                const w = cluster.fork()
+                w.send({ missed, factory, chunk_size, key })
+                w.on('message', p => {
+                    const id = p[0]
+                    pairs[id] = { id, pair: p[1], token0: p[2], token1: p[3] }
+                    if (progress) progress(++progress_i, progress_end)
                     if (filename) {
-                        var pair
-                        while (pair = pairs[next_pair_order]) {
-                            fs.appendFileSync(filename, pair.id + ',' + pair.pair + ',' + pair.token0 + ',' + pair.token1 + '\n')
+                        var _
+                        while (_ = pairs[next_pair_order]) {
+                            fs.appendFileSync(filename, `${_.id},${_.pair},${_.token0},${_.token1}\n`)
                             next_pair_order++
                         }
                     }
                 })
-                loader.on('close', y)
+                w.on('exit', y)
             }))
-        )
-        .then(() => {
-            fs.unlinkSync(jobs_data_filename)
-            return pairs
-        })
+        ).then(() => pairs)
     })
 }
 
 
+module.exports.clear_cache = () =>
+    fs.unlinkSync(default_filename)
+
 module.exports.all = (params = {}) =>
     load(params)
+    
+module.exports.count = () =>
+    load({count: true})
 
 module.exports.onupdate = function onupdate(callback, params = {}) {
     var subscribe = true, timeout

package/loader.js

@@ -1,6 +1,5 @@
 const { parseAbiItem, createPublicClient, http } = require('viem')
 const { mainnet } = require('viem/chains')
-const fs = require('fs')
 
 const POOL = {
     ID: 0,
@@ -69,7 +68,7 @@ async function load(params) {
                     pool[POOL.TOKEN0] = token0_result.result.toLowerCase()
                     pool[POOL.TOKEN1] = token1_result.result.toLowerCase()
 
-                    console.log(pool.join(','))
+                    process.send(pool)
                 } else {
                     retry_missed.push(pools_ok[j].id)
                 }
@@ -88,17 +87,11 @@ async function load(params) {
     }
 }
 
-const jobs_data_filename = process.argv[2]
-const job_index = +process.argv[3]
+process.on('message', jobs_data => {
+    const client = createPublicClient({
+        chain: mainnet,
+        transport: http(`https://eth-mainnet.g.alchemy.com/v2/${jobs_data.key}`)
+    })
 
-if (isNaN(job_index)) process.exit(1)
-
-const jobs_data = JSON.parse(fs.readFileSync(jobs_data_filename, 'utf8'))
-jobs_data.missed = jobs_data.missed[job_index]
-
-const client = createPublicClient({
-    chain: mainnet,
-    transport: http(`https://eth-mainnet.g.alchemy.com/v2/${jobs_data.key}`)
+    load({client, ...jobs_data}).then(() => process.exit())
 })
-
-load({client, ...jobs_data})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "uniswap-v2-loader",
-  "version": "1.2.0",
+  "version": "1.4.0",
   "description": "Uniswap v2 protocol loader",
   "keywords": [
     "uniswap-v2",
@@ -24,6 +24,9 @@
   "scripts": {
     "test": "node --test test.js"
   },
+  "bin": {
+    "uniswap-v2-loader": "./bin/uniswap-v2-loader"
+  },
   "dependencies": {
     "viem": "^2.46.2"
   }

package/test.js

@@ -46,3 +46,10 @@ test('Heavy test load first 3000 pairs', () =>
         assert.equal(pairs.length, 3000)
     })
 )
+
+test('Each line at CSV cache file should be orderd by pair id (factory id)', () => {
+    const lines = fs.readFileSync(require('./default_cache_filename'), 'utf8').trim().split('\n')
+    for (var i = 0; i < lines.length; i++)
+        assert.equal(i, +lines[i].split(',').shift())
+})
+