uniswap-v2-loader 5.0.18 → 5.0.20

package/.github/workflows/test.yml

@@ -35,7 +35,7 @@ jobs:
 
       run: npm i -g .
 
-    - name: CLI Load first 4 pairs from Uniswap V2 using 2 workers
+    - name: Test CLI load first 4 pairs from Uniswap V2 using 2 workers
       run: uniswap-v2-loader --to=4 --multicall_size=2 --workers=2
 
     - name: Run ESM tests

package/README.md

@@ -41,6 +41,7 @@ High-performance parallel fetcher for liquidity pairs. Efficiently synchronizes
 | `multicall_size` | `number` | RPC batch size per multicall request. | `50` |
 | `workers` | `number` | Number of parallel worker threads. | `CPU - 1` |
 | `progress` | `function` | Progress callback: `(current, total) => {}`. | `undefined` |
+| `abort_signal` | `AbortSignal` | Signal to cancel loading and release workers. | `undefined` |
 
 **Returns**: `Promise<Pair[]>`
 

package/index.d.ts

@@ -36,6 +36,8 @@ export interface load_params {
   pairs?: pair[]
   /** Timeout between updates in milliseconds. */
   update_timeout?: number
+  /** Signal to abort the loading process. */
+  abort_signal?: AbortSignal
 }
 
 /**

package/index.js

@@ -16,9 +16,11 @@ const load = (params = {}) => {
         from = 0,
         to,
         progress,
+        abort_signal,
         workers = max_workers,
         pairs,
     } = params
+
     filename ??= default_cache_filename(factory)
     workers = Math.min(workers, max_workers)
 
@@ -44,10 +46,11 @@ const load = (params = {}) => {
 
         return Promise.resolve(pairs.slice(0, to))
     }
-    
+
     return (to
         ? Promise.resolve(to)
         : fetch('https://eth-mainnet.g.alchemy.com/v2/' + key, {
+            signal: abort_signal,
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({
@@ -92,18 +95,14 @@ const load = (params = {}) => {
             const ids = []
             for (var i = start_loading_from; i < all_pairs_length; i++)
                 ids.push(i)
-            return require('./loader')({ ids, factory, key, multicall_size }, onpair)
+            return require('./loader')({ ids, factory, key, multicall_size, abort_signal }, onpair)
             .then(() => pairs)
         }
 
-        const missed = []
-        for (var i = start_loading_from, ids; i < all_pairs_length; i++) {
-            if (!ids || ids.length % multicall_size == 0) {
-                ids = []
-                missed.push(ids)
-            }
-            ids.push(i)
-        }
+        const missed = Array(workers).fill(null).map(() => [])
+
+        for (var i = start_loading_from, iw = 0; i < all_pairs_length; i++)
+            missed[iw++ % workers].push(i)
         
         cluster.setupPrimary({ exec: path.join(__dirname, 'loader.js') })
         
@@ -111,14 +110,20 @@ const load = (params = {}) => {
             missed
             .filter(_ => _.length)
             .map((ids, i) => new Promise(y => {
+                if (abort_signal?.aborted) return y()
                 const w = cluster.fork()
+                const onabort = () => w.send('abort')
+                abort_signal?.addEventListener('abort', onabort, { once: true })
                 w.send({ ids, factory, key, multicall_size })
                 w.on('message', onpair)
-                w.on('exit', y)
+                w.on('exit', () => {
+                    abort_signal?.removeEventListener('abort', onabort)
+                    y()
+                })
             }))
         ).then(() => pairs)
     })
-    .catch(() => new Promise(resolve => setTimeout(() => resolve(load(params)), 1000)))
+    .catch(() => abort_signal?.aborted ? pairs : new Promise(resolve => setTimeout(() => resolve(load(params)), 1000)))
 }
 
 module.exports.load = (params = {}) =>

package/loader.js

@@ -1,6 +1,7 @@
-const get_pairs_addresses = (key, factory, ids) => ids.length == 0
+const get_pairs_addresses = (key, factory, ids, abort_signal) => ids.length == 0
     ? Promise.resolve([])
     : fetch('https://eth-mainnet.g.alchemy.com/v2/' + key, {
+        signal: abort_signal,
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify(ids.map((id, i) => ({
@@ -30,14 +31,15 @@ const get_pairs_addresses = (key, factory, ids) => ids.length == 0
         return failed_ids.length == 0
             ? addresses
             : new Promise(resolve => setTimeout(() => resolve(
-                get_pairs_addresses(key, factory, failed_ids).then(retried => [...addresses, ...retried])
+                get_pairs_addresses(key, factory, failed_ids, abort_signal).then(retried => [...addresses, ...retried])
             ), 10000))
     })
-    .catch(() => new Promise(resolve => setTimeout(() => resolve(get_pairs_addresses(key, factory, ids)), 10000)))
+    .catch(() => abort_signal?.aborted ? [] : new Promise(resolve => setTimeout(() => resolve(get_pairs_addresses(key, factory, ids, abort_signal)), 10000)))
 
-const get_tokens = (key, addresses) => addresses.length == 0
+const get_tokens = (key, addresses, abort_signal) => addresses.length == 0
     ? Promise.resolve({})
     : fetch('https://eth-mainnet.g.alchemy.com/v2/' + key, {
+        signal: abort_signal,
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify(addresses.flatMap((address, i) => [
@@ -83,20 +85,20 @@ const get_tokens = (key, addresses) => addresses.length == 0
         return failed_addresses.length == 0
             ? tokens
             : new Promise(resolve => setTimeout(() => resolve(
-                get_tokens(key, failed_addresses).then(retried => ({ ...tokens, ...retried }))
+                get_tokens(key, failed_addresses, abort_signal).then(retried => ({ ...tokens, ...retried }))
             ), 10000))
     })
-    .catch(() => new Promise(resolve => setTimeout(() => resolve(get_tokens(key, addresses)), 10000)))
+    .catch(() => abort_signal?.aborted ? {} : new Promise(resolve => setTimeout(() => resolve(get_tokens(key, addresses, abort_signal)), 10000)))
 
-const main = ({ids, factory, key, multicall_size}, onpair) => {
+const main = ({ids, factory, key, multicall_size, abort_signal}, onpair) => {
     const chunks = []
     for (let i = 0; i < ids.length; i += multicall_size)
         chunks.push(ids.slice(i, i + multicall_size))
 
     return chunks.reduce((p, ids, ic) =>
         p.then(() =>
-            get_pairs_addresses(key, factory, ids).then(pairs_addresses =>
-                get_tokens(key, pairs_addresses).then(tokens =>
+            get_pairs_addresses(key, factory, ids, abort_signal).then(pairs_addresses =>
+                get_tokens(key, pairs_addresses, abort_signal).then(tokens =>
                     ids.forEach((id, i) =>
                         onpair({
                             id,
@@ -113,12 +115,20 @@ const main = ({ids, factory, key, multicall_size}, onpair) => {
 }
 
 
-if (require.main != module)
+if (require.main != module) {
     module.exports = main
-else
+} else {
+    const abort_controller = new AbortController()
+    const abort_signal = abort_controller.signal
+
     process.on(
         'message',
-        message =>
-            main(message, pair => process.send(pair))
-            .then(() => process.exit())
+        message => {
+            if (message == 'abort')
+                abort_controller.abort()
+            else
+                main({...message, abort_signal}, pair => process.send(pair))
+                .finally(() => process.exit())
+        }
     )
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "uniswap-v2-loader",
-  "version": "5.0.18",
+  "version": "5.0.20",
   "description": "Uniswap v2 protocol loader",
   "keywords": [
     "uniswap-v2",

package/test-esm.mjs

@@ -1,18 +0,0 @@
-import { load, subscribe } from './index.mjs'
-import assert from 'node:assert/strict'
-import { describe, it } from 'node:test'
-
-describe('ESM Support', () => {
-    it('should import load and subscribe', () => {
-        assert.equal(typeof load, 'function')
-        assert.equal(typeof subscribe, 'function')
-    })
-
-    it('should load first pair', () => 
-        load({ to: 1 })
-        .then(pairs => {
-            assert.equal(pairs.length, 1)
-            assert.equal(pairs[0].id, 0)
-        })
-    )
-})