ponder 0.9.4 → 0.9.5-debug.1

package/src/sync-historical/index.ts

@@ -1,4 +1,5 @@
 import type { Common } from "@/internal/common.js";
+import { ShutdownError } from "@/internal/errors.js";
 import type {
   BlockFilter,
   Factory,
@@ -109,18 +110,17 @@ export const createHistoricalSync = async (
   >();
 
   /**
-   * Data about the range passed to "eth_getLogs" for all log
-   *  filters and log factories.
+   * Data about the range passed to "eth_getLogs" share among all log
+   * filters and log factories.
    */
-  const getLogsRequestMetadata = new Map<
-    LogFilter | LogFactory,
-    {
-      /** Estimate optimal range to use for "eth_getLogs" requests */
-      estimatedRange: number;
-      /** Range suggested by an error message */
-      confirmedRange?: number;
-    }
-  >();
+  let logsRequestMetadata: {
+    /** Estimate optimal range to use for "eth_getLogs" requests */
+    estimatedRange: number;
+    /** Range suggested by an error message */
+    confirmedRange?: number;
+  } = {
+    estimatedRange: 500,
+  };
   /**
    * Intervals that have been completed for all filters in `args.sources`.
    *
@@ -164,13 +164,12 @@ export const createHistoricalSync = async (
   }): Promise<SyncLog[]> => {
     //  Use the recommended range if available, else don't chunk the interval at all.
 
-    const metadata = getLogsRequestMetadata.get(filter);
-    const intervals = metadata
-      ? getChunks({
-          interval,
-          maxChunkSize: metadata.confirmedRange ?? metadata.estimatedRange,
-        })
-      : [interval];
+    const intervals = getChunks({
+      interval,
+      maxChunkSize:
+        logsRequestMetadata.confirmedRange ??
+        logsRequestMetadata.estimatedRange,
+    });
 
     const topics =
       "eventSelector" in filter
@@ -254,12 +253,12 @@ export const createHistoricalSync = async (
               }', updating recommended range to ${range}.`,
             });
 
-            getLogsRequestMetadata.set(filter, {
+            logsRequestMetadata = {
               estimatedRange: range,
               confirmedRange: getLogsErrorResponse.isSuggestedRange
                 ? range
                 : undefined,
-            });
+            };
 
             return syncLogsDynamic({ address, interval, filter });
           }),
@@ -285,12 +284,9 @@ export const createHistoricalSync = async (
      * error has been received but the error didn't suggest a range.
      */
 
-    if (
-      getLogsRequestMetadata.has(filter) &&
-      getLogsRequestMetadata.get(filter)!.confirmedRange === undefined
-    ) {
-      getLogsRequestMetadata.get(filter)!.estimatedRange = Math.round(
-        getLogsRequestMetadata.get(filter)!.estimatedRange * 1.05,
+    if (logsRequestMetadata.confirmedRange === undefined) {
+      logsRequestMetadata.estimatedRange = Math.round(
+        logsRequestMetadata.estimatedRange * 1.05,
       );
     }
 
@@ -819,6 +815,10 @@ export const createHistoricalSync = async (
           } catch (_error) {
             const error = _error as Error;
 
+            if (args.common.shutdown.isKilled) {
+              throw new ShutdownError();
+            }
+
             args.common.logger.error({
               service: "sync",
               msg: `Fatal error: Unable to sync '${args.network.name}' from ${interval[0]} to ${interval[1]}.`,

package/src/sync-realtime/index.ts

@@ -32,6 +32,7 @@ import type {
   SyncTransactionReceipt,
 } from "@/types/sync.js";
 import { mutex } from "@/utils/mutex.js";
+import type { Queue } from "@/utils/queue.js";
 import { range } from "@/utils/range.js";
 import type { RequestQueue } from "@/utils/requestQueue.js";
 import {
@@ -44,7 +45,6 @@ import {
 } from "@/utils/rpc.js";
 import { startClock } from "@/utils/timer.js";
 import { wait } from "@/utils/wait.js";
-import type { Queue } from "@ponder/common";
 import { type Address, type Hash, hexToNumber, zeroHash } from "viem";
 import { isFilterInBloom, zeroLogsBloom } from "./bloom.js";
 

package/src/utils/dedupe.ts

@@ -0,0 +1,40 @@
+/**
+ * Remove duplicate values from an array.
+ *
+ * @param getId callback used to calculate a unique value for an element of the array.
+ *
+ * @example
+ * dedupe([1,1,2,3]) // [1,2,3]
+ *
+ * dedupe(
+ *   [
+ *     { a: 1, b: 2 },
+ *     { a: 1, b: 2 },
+ *     { a: 2, b: 2 },
+ *   ],
+ *   (e) => `${e.a}_${e.b}`,
+ * ) // [{a: 1, b: 2}, {a: 2, b: 2}]
+ *
+ */
+export function dedupe<item, id>(
+  arr: item[] | readonly item[],
+  getId?: (x: item) => id,
+): item[] {
+  const seen = new Set<id | item>();
+
+  return arr.filter((x) => {
+    if (seen.has(getId ? getId(x) : x)) return false;
+
+    seen.add(x);
+    return true;
+  });
+}
+
+dedupe(
+  [
+    { a: 1, b: 2 },
+    { a: 1, b: 2 },
+    { a: 2, b: 2 },
+  ],
+  (e) => `${e.a}_${e.b}`,
+);

package/src/utils/generators.ts

@@ -1,4 +1,4 @@
-import { promiseWithResolvers } from "@ponder/common";
+import { promiseWithResolvers } from "@/utils/promiseWithResolvers.js";
 
 /**
  * Merges multiple async generators into a single async generator.

package/src/utils/mutex.ts

@@ -1,4 +1,4 @@
-import { type Queue, createQueue } from "@ponder/common";
+import { type Queue, createQueue } from "@/utils/queue.js";
 
 export type Mutex<T, P> = ((params: T) => Promise<P>) & Queue<P, T>;
 

package/src/utils/promiseWithResolvers.ts

@@ -0,0 +1,20 @@
+export type PromiseWithResolvers<TPromise> = {
+  resolve: (arg: TPromise) => void;
+  reject: (error: Error) => void;
+  promise: Promise<TPromise>;
+};
+
+/**
+ * @description Application level polyfill.
+ */
+export const promiseWithResolvers = <
+  TPromise,
+>(): PromiseWithResolvers<TPromise> => {
+  let resolve: (arg: TPromise) => void;
+  let reject: (error: Error) => void;
+  const promise = new Promise<TPromise>((_resolve, _reject) => {
+    resolve = _resolve;
+    reject = _reject;
+  });
+  return { resolve: resolve!, reject: reject!, promise };
+};

package/src/utils/queue.ts

@@ -0,0 +1,250 @@
+import {
+  type PromiseWithResolvers,
+  promiseWithResolvers,
+} from "./promiseWithResolvers.js";
+
+export type InnerQueue<returnType, taskType> = {
+  task: taskType;
+  resolve: (arg: returnType) => void;
+  reject: (error: Error) => void;
+}[];
+
+export type Queue<returnType, taskType> = {
+  size: () => number;
+  pending: () => Promise<number>;
+  add: (task: taskType) => Promise<returnType>;
+  clear: () => void;
+  isStarted: () => boolean;
+  start: () => Promise<void>;
+  pause: () => void;
+  onIdle: () => Promise<void>;
+  onEmpty: () => Promise<void>;
+  setParameters: (
+    parameters: Pick<
+      CreateQueueParameters<unknown, unknown>,
+      "frequency" | "concurrency"
+    >,
+  ) => void;
+};
+
+export type CreateQueueParameters<returnType, taskType> = {
+  worker: (task: taskType) => Promise<returnType>;
+  initialStart?: boolean;
+  browser?: boolean;
+} & (
+  | {
+      concurrency: number;
+      frequency: number;
+    }
+  | { concurrency: number; frequency?: undefined }
+  | { concurrency?: undefined; frequency: number }
+);
+
+const validateParameters = ({
+  concurrency,
+  frequency,
+}: Pick<
+  CreateQueueParameters<unknown, unknown>,
+  "frequency" | "concurrency"
+>) => {
+  if (concurrency === undefined && frequency === undefined) {
+    throw new Error(
+      "Invalid queue configuration, must specify either 'concurrency' or 'frequency'.",
+    );
+  }
+
+  if (concurrency !== undefined && concurrency <= 0) {
+    throw new Error(
+      `Invalid value for queue 'concurrency' option. Got ${concurrency}, expected a number greater than zero.`,
+    );
+  }
+
+  if (frequency !== undefined && frequency <= 0) {
+    throw new Error(
+      `Invalid value for queue 'frequency' option. Got ${frequency}, expected a number greater than zero.`,
+    );
+  }
+};
+
+export const createQueue = <returnType, taskType = void>({
+  worker,
+  initialStart = false,
+  browser = true,
+  ..._parameters
+}: CreateQueueParameters<returnType, taskType>): Queue<
+  returnType,
+  taskType
+> => {
+  validateParameters(_parameters);
+
+  const parameters: Pick<
+    CreateQueueParameters<unknown, unknown>,
+    "frequency" | "concurrency"
+  > = _parameters;
+  let queue = new Array<InnerQueue<returnType, taskType>[number]>();
+  let pending = 0;
+  let timestamp = 0;
+  let requests = 0;
+  let isStarted = initialStart;
+
+  let timer: NodeJS.Timeout | undefined;
+
+  let emptyPromiseWithResolvers:
+    | (PromiseWithResolvers<void> & { completed: boolean })
+    | undefined = undefined;
+  let idlePromiseWithResolvers:
+    | (PromiseWithResolvers<void> & { completed: boolean })
+    | undefined = undefined;
+
+  const next = () => {
+    if (!isStarted) return;
+
+    const _timestamp = Date.now();
+
+    if (Math.floor(_timestamp / 1_000) !== timestamp) {
+      requests = 0;
+      timestamp = Math.floor(_timestamp / 1_000);
+    }
+
+    if (timer) return;
+
+    while (
+      (parameters.frequency !== undefined
+        ? requests < parameters.frequency
+        : true) &&
+      (parameters.concurrency !== undefined
+        ? pending < parameters.concurrency
+        : true) &&
+      queue.length > 0
+    ) {
+      const { task, resolve, reject } = queue.shift()!;
+
+      requests++;
+      pending++;
+
+      worker(task)
+        .then(resolve)
+        .catch(reject)
+        .finally(() => {
+          pending--;
+
+          if (
+            idlePromiseWithResolvers !== undefined &&
+            queue.length === 0 &&
+            pending === 0
+          ) {
+            idlePromiseWithResolvers.resolve();
+            idlePromiseWithResolvers.completed = true;
+          }
+
+          browser ? next() : process.nextTick(next);
+        });
+
+      if (emptyPromiseWithResolvers !== undefined && queue.length === 0) {
+        emptyPromiseWithResolvers.resolve();
+        emptyPromiseWithResolvers.completed = true;
+      }
+    }
+
+    if (
+      parameters.frequency !== undefined &&
+      requests >= parameters.frequency
+    ) {
+      timer = setTimeout(
+        () => {
+          timer = undefined;
+          next();
+        },
+        1_000 - (_timestamp % 1_000),
+      );
+      return;
+    }
+  };
+
+  return {
+    size: () => queue.length,
+    pending: () => {
+      if (browser) {
+        return new Promise<number>((resolve) =>
+          setTimeout(() => resolve(pending)),
+        );
+      } else {
+        return new Promise<number>((resolve) =>
+          setImmediate(() => resolve(pending)),
+        );
+      }
+    },
+    add: (task: taskType) => {
+      const { promise, resolve, reject } = promiseWithResolvers<returnType>();
+      queue.push({ task, resolve, reject });
+
+      next();
+
+      return promise;
+    },
+    clear: () => {
+      queue = new Array<InnerQueue<returnType, taskType>[number]>();
+      clearTimeout(timer);
+      timer = undefined;
+    },
+    isStarted: () => isStarted,
+    start: () => {
+      if (browser) {
+        return new Promise<number>((resolve) =>
+          setTimeout(() => resolve(pending)),
+        ).then(() => {
+          isStarted = true;
+          next();
+        });
+      } else {
+        return new Promise<number>((resolve) =>
+          process.nextTick(() => resolve(pending)),
+        ).then(() => {
+          isStarted = true;
+          next();
+        });
+      }
+    },
+    pause: () => {
+      isStarted = false;
+    },
+    onIdle: () => {
+      if (
+        idlePromiseWithResolvers === undefined ||
+        idlePromiseWithResolvers.completed
+      ) {
+        if (queue.length === 0 && pending === 0) return Promise.resolve();
+
+        idlePromiseWithResolvers = {
+          ...promiseWithResolvers<void>(),
+          completed: false,
+        };
+      }
+      return idlePromiseWithResolvers.promise;
+    },
+    onEmpty: () => {
+      if (
+        emptyPromiseWithResolvers === undefined ||
+        emptyPromiseWithResolvers.completed
+      ) {
+        if (queue.length === 0) return Promise.resolve();
+
+        emptyPromiseWithResolvers = {
+          ...promiseWithResolvers<void>(),
+          completed: false,
+        };
+      }
+      return emptyPromiseWithResolvers.promise;
+    },
+    setParameters: (_parameters) => {
+      validateParameters(_parameters);
+
+      if ("frequency" in _parameters) {
+        parameters.frequency = _parameters.frequency;
+      }
+      if ("concurrency" in _parameters) {
+        parameters.concurrency = _parameters.concurrency;
+      }
+    },
+  } as Queue<returnType, taskType>;
+};

package/src/utils/requestQueue.ts

@@ -1,7 +1,7 @@
 import type { Common } from "@/internal/common.js";
 import { ShutdownError } from "@/internal/errors.js";
 import type { Network } from "@/internal/types.js";
-import { type Queue, createQueue } from "@ponder/common";
+import { type Queue, createQueue } from "@/utils/queue.js";
 import {
   type GetLogsRetryHelperParameters,
   getLogsRetryHelper,