uniswap-v2-loader 1.1.0 → 1.2.0

package/CHANGELOG.md

@@ -0,0 +1,2 @@
+## [1.2.0] - 2026-02-22
+- Added `onupdate` function for subscribing to new pairs.

package/README.md

@@ -7,12 +7,26 @@ The package uses Alchemy. Set your key as an environment variable (a default key
 `export KEY=your_alchemy_key`
 
 ## API Reference
-### `all()`
-- **Description**: Fetches all token pairs from the Uniswap v2 factory. It utilizes multicall from `viem` and splits the loading process between multiple CPUs for high-speed execution.
-- **Returns**: `Promise<Array<Object>>`
+### `all(params)`
+- **Description**: Fetches token pairs from the Uniswap v2 factory. It utilizes multicall from `viem` and splits the loading process between multiple CPUs for high-speed execution.
+- **Arguments**:
+    - `params`: (Object)
+        - `from`: (number) Start loading from this index (default 0).
+        - `to`: (number) Load up to this index.
+        - `filename`: (string) Path to cache CSV file.
+        - `chunk_size`: (number) Items per multicall (default 50).
+- **Returns**: `Promise<Array<Object>>` (Array of Pair Objects)
 
-### Output Format
-The function returns a `Promise` resolving to an array of objects with the following fields:
+### `onupdate(callback, params)`
+- **Description**: Subscribes to new pairs appearing at the factory contract. It initially calls the callback with cached pairs and then polls for updates.
+- **Arguments**:
+    - `callback`: (function) Called with an array of Pair Objects.
+    - `params`: (Object) Same as `all()` plus:
+        - `update_timeout`: (number) Polling interval in ms (default 5000).
+- **Returns**: `Function` An unsubscribe function to stop polling.
+
+### Pair Object
+The pair object contains the following fields:
 - `id`: (number) The pair index.
 - `pair`: (string) The pair contract address.
 - `token0`: (string) The address of the first token in the pair.
@@ -20,11 +34,18 @@ The function returns a `Promise` resolving to an array of objects with the follo
 
 ## Usage
 ```javascript
-const { all } = require('uniswap-v2-loader')
+const { all, onupdate } = require('uniswap-v2-loader')
 
-all().then(pairs => 
-    pairs.forEach(({id, pair, token0, token1}) => 
-        console.log(`ID: ${id} | Pair: ${pair} | Tokens: ${token0}, ${token1}`)
-    )
+// Load initial set
+all({to: 10}).then(pairs =>
+    pairs.forEach(({id, pair}) => console.log(`ID: ${id} | Pair: ${pair}`))
 )
+
+// Subscribe to new pairs (24/7 monitoring)
+const unsubscribe = onupdate(pairs => {
+    console.log(`Pools count: ${pairs.length}`)
+})
+
+// Stop monitoring if needed
+// unsubscribe()
 ```

package/index.js

@@ -1,8 +1,26 @@
 const { spawn } = require('child_process')
-const os = require('os')
 const fs = require('fs')
+const os = require('os')
+const path = require('path')
+const env = process.env
+const home = os.homedir()
+const pkg = require('./package.json')
+const default_filename = path.join(
+  ...(process.platform === 'win32'
+      ? (env.LOCALAPPDATA || env.APPDATA)
+        ? [env.LOCALAPPDATA || env.APPDATA]
+        : [home, 'AppData', 'Local']
+      : process.platform === 'darwin'
+        ? [home, 'Library', 'Caches']
+        : (env.XDG_CACHE_HOME && path.isAbsolute(env.XDG_CACHE_HOME))
+          ? [env.XDG_CACHE_HOME]
+          : [home, '.cache']
+  ),
+  pkg.name + '_pairs.csv'
+)
 const { parseAbiItem, createPublicClient, http } = require('viem')
 const { mainnet } = require('viem/chains')
+
 const workers = os.cpus().length - 1
 const missed = Array(workers).fill(null).map(() => [])
 const key = process.env.KEY || 'FZBvlPrOxtgaKBBkry3SH0W1IqH4Y5tu'
@@ -13,12 +31,24 @@ const client = createPublicClient({
 })
 
 const load = params => {
-    const {filename, to} = params
-    const pairs_ids = filename && fs.existsSync(filename)
+    const {filename = default_filename, to, from = 0, chunk_size = 50} = params
+    const pairs = params.pairs || fs.existsSync(filename)
         ? fs.readFileSync(filename).toString().trim().split('\n')
-            .map(line => +line.split(',')[0])
+            .reduce((pairs, line) => {
+                line = line.split(',')
+                const id = +line[0]
+                if (id >= from && (to == undefined || id <= to)) pairs.push({
+                    id,
+                    pair: line[1],
+                    token0: line[2],
+                    token1: line[3]
+                })
+                return pairs
+            }, [])
         : []
 
+    if (pairs.length > to) return Promise.resolve(pairs.slice(0, to))
+
     return (to
         ? Promise.resolve(to)
         : client.readContract({
@@ -27,23 +57,24 @@ const load = params => {
             functionName: 'allPairsLength'
         }).then(_ => Number(_))
     ).then(allPairsLength => {
-        const pairs = Array(allPairsLength)
         var next_pair_order = 0    
-        var start_from = pairs_ids.length
-            ? pairs_ids[pairs_ids - 1] + 1
+        const start_loading_from = pairs.length
+            ? Math.max(from || 0, pairs[pairs.length - 1].id + 1)
             : 0
 
         missed.forEach(_ => _.length = 0)
         
-        for (var i = start_from, rr = 0; i < allPairsLength; i++) {
+        for (var i = start_loading_from, rr = 0; i < allPairsLength; i++) {
             missed[rr].push(i)
-            rr = (rr + 1) % workers
+            if (missed[rr].length % chunk_size == 0)
+                rr = (rr + 1) % workers
         }
-                
+        
         const jobs_data_filename = `jobs_data_${Date.now()}.json`
         fs.writeFileSync(jobs_data_filename, JSON.stringify({
             missed,
             factory,
+            chunk_size,
             key
         }), 'utf8')
         
@@ -52,16 +83,21 @@ const load = params => {
             .filter(_ => _.length)
             .map((_, i) => new Promise(y => {
                 const loader = spawn('node', ['loader.js', jobs_data_filename, i.toString()])
-                loader.stdout.on('data', line => {
-                    line = line.toString()
-                    const data = line.trim().split(',')
-                    const id = data[0]
-                    pairs[id] = {
-                        id,
-                        pair: data[1],
-                        token0: data[2],
-                        token1: data[3]
-                    }
+                loader.stdout.on('data', data => {
+                    data += data.toString()
+                    if (!data.includes('\n')) return
+                    const lines = data.split('\n')
+                    data = lines.shift()
+                    lines.forEach(line => {
+                        const a = line.split(',')
+                        const id = +a[0]
+                        pairs[id] = {
+                            id,
+                            pair: a[1],
+                            token0: a[2],
+                            token1: a[3]
+                        }
+                    })
                     if (filename) {
                         var pair
                         while (pair = pairs[next_pair_order]) {
@@ -81,5 +117,36 @@ const load = params => {
 }
 
 
-module.exports.all = params =>
+module.exports.all = (params = {}) =>
+    load(params)
+
+module.exports.onupdate = function onupdate(callback, params = {}) {
+    var subscribe = true, timeout
     load(params)
+    .then(pairs => {
+        callback(pairs)
+
+        const update = pairs =>
+            timeout = setTimeout(
+                () =>
+                    load({...params, pairs, from: pairs.length})
+                    .then(pairs => {
+                        if (!subscribe) return
+                        callback(pairs)
+                        if (!subscribe) return
+                        if (params.to && pairs[pairs.length - 1].id >= params.to) return
+                        update(pairs)
+                    }),
+                params.update_timeout || 5000
+            )
+
+        if (!subscribe) return
+        if (params.to && pairs[pairs.length - 1].id >= params.to) return
+        update(pairs)
+    })
+
+    return () => {
+        subscribe = false
+        if (timeout) clearTimeout(timeout)
+    }
+}

package/loader.js

@@ -68,14 +68,17 @@ async function load(params) {
                     pool[POOL.ADDRESS] = pools_ok[j].address.toLowerCase()
                     pool[POOL.TOKEN0] = token0_result.result.toLowerCase()
                     pool[POOL.TOKEN1] = token1_result.result.toLowerCase()
-                    
+
                     console.log(pool.join(','))
                 } else {
                     retry_missed.push(pools_ok[j].id)
                 }
             }
-        } catch (error) {
-            console.error(error.message)
+        } catch (_) {
+            chunk.forEach(id => {
+                if (retry_missed.includes(id)) return
+                retry_missed.push(id)
+            })
         }
     }
     
@@ -88,7 +91,7 @@ async function load(params) {
 const jobs_data_filename = process.argv[2]
 const job_index = +process.argv[3]
 
-if (isNaN(job_index)) console.error('Required pass an index of job via argument') || process.exit(1)
+if (isNaN(job_index)) process.exit(1)
 
 const jobs_data = JSON.parse(fs.readFileSync(jobs_data_filename, 'utf8'))
 jobs_data.missed = jobs_data.missed[job_index]
@@ -98,4 +101,4 @@ const client = createPublicClient({
     transport: http(`https://eth-mainnet.g.alchemy.com/v2/${jobs_data.key}`)
 })
 
-load({client, ...jobs_data, chunk_size: 50})
+load({client, ...jobs_data})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "uniswap-v2-loader",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Uniswap v2 protocol loader",
   "keywords": [
     "uniswap-v2",

package/test.js

@@ -28,4 +28,21 @@ test('Load first two pairs to file', () => {
     .finally(() =>
         fs.unlinkSync(filename)
     )
-})
+})
+
+test('onupdate should call provided callback with 2 pairs for a current moment (from cache)', () => {
+    return new Promise(y => {
+        const unsubscribe = uniswap_v2_loader.onupdate(pairs => {
+            assert.equal(pairs.length, 2)
+            unsubscribe()
+            y()
+        }, {to: 2})
+    })
+})
+
+test('Heavy test load first 3000 pairs', () =>
+    uniswap_v2_loader.all({to: 3000})
+    .then(pairs => {
+        assert.equal(pairs.length, 3000)
+    })
+)