soroban-events 1.0.0 → 2.0.0

package/LICENSE

@@ -0,0 +1,23 @@
+MIT License
+
+Copyright (c) 2026 poaspergillus
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -1,19 +1,28 @@
 {
   "name": "soroban-events",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "description": "Resilient, windowed event streamer and XDR decoder for Soroban RPC",
   "type": "module",
   "main": "./src/index.js",
   "exports": {
-    ".": "./src/index.js"
+    ".": {
+      "types": "./src/index.d.ts",
+      "import": "./src/index.js"
+    }
   },
   "files": [
-    "src/"
+    "src/",
+    "README.md",
+    "LICENSE"
   ],
   "scripts": {
     "test": "node --test tests/*.test.js",
     "test:live": "node tests/test-engine.js",
-    "pack:check": "npm pack --dry-run"
+    "pack:check": "npm pack --dry-run",
+    "cli": "node src/cli.js",
+    "test:unit": "node --test tests/*.test.js",
+    "audit": "node scripts/audit.mjs",
+    "pack:audit": "node scripts/pack-audit.mjs"
   },
   "keywords": [
     "soroban",
@@ -22,7 +31,11 @@
     "event-streaming",
     "event-indexer",
     "xdr",
-    "rpc"
+    "rpc",
+    "soroban-rpc",
+    "blockchain-events",
+    "event-replay",
+    "event-backfill"
   ],
   "license": "MIT",
   "engines": {
@@ -30,5 +43,9 @@
   },
   "dependencies": {
     "@stellar/stellar-sdk": "^17.1.0"
-  }
+  },
+  "bin": {
+    "soroban-events": "./src/cli.js"
+  },
+  "types": "./src/index.d.ts"
 }

package/src/backfill.js

@@ -0,0 +1,451 @@
+import { assertEventStore } from './store.js';
+export function orderBackfillEvents(windowResults) {
+  if (!Array.isArray(windowResults)) {
+    throw new TypeError(
+      'window results must be an array'
+    );
+  }
+
+  const events = [];
+
+  for (const result of windowResults) {
+    if (!Array.isArray(result)) {
+      throw new TypeError(
+        'each window result must be an array'
+      );
+    }
+
+    events.push(...result);
+  }
+
+  events.sort(compareEvents);
+
+  const seen = new Set();
+  const output = [];
+
+  for (const event of events) {
+    const id = event?.id;
+
+    if (id != null) {
+      if (seen.has(id)) continue;
+      seen.add(id);
+    }
+
+    output.push(event);
+  }
+
+  return output;
+}
+
+export function compareEvents(a, b) {
+  const ledgerA = Number(a?.ledger ?? 0);
+  const ledgerB = Number(b?.ledger ?? 0);
+
+  if (ledgerA !== ledgerB) {
+    return ledgerA - ledgerB;
+  }
+
+  const txA = Number(
+    a?.transactionIndex ??
+    a?.txIndex ??
+    0
+  );
+
+  const txB = Number(
+    b?.transactionIndex ??
+    b?.txIndex ??
+    0
+  );
+
+  if (txA !== txB) {
+    return txA - txB;
+  }
+
+  const opA = Number(
+    a?.operationIndex ??
+    a?.opIndex ??
+    0
+  );
+
+  const opB = Number(
+    b?.operationIndex ??
+    b?.opIndex ??
+    0
+  );
+
+  if (opA !== opB) {
+    return opA - opB;
+  }
+
+  return String(a?.id ?? "").localeCompare(
+    String(b?.id ?? "")
+  );
+}
+
+export class BackfillError extends Error {
+  constructor(message, options = {}) {
+    super(message, options);
+    this.name = 'BackfillError';
+  }
+}
+
+export class BackfillEngine {
+  constructor(streamer, options = {}) {
+    if (
+      !streamer ||
+      typeof streamer.getEventsWindowed !== 'function'
+    ) {
+      throw new TypeError(
+        'backfill streamer must implement getEventsWindowed()'
+      );
+    }
+
+    this.streamer = streamer;
+    this.concurrency = options.concurrency ?? 1;
+    this.windowSize =
+      options.windowSize ??
+      streamer.windowSize ??
+      9000;
+
+    if (
+      !Number.isSafeInteger(this.concurrency) ||
+      this.concurrency < 1
+    ) {
+      throw new TypeError(
+        'backfill concurrency must be a positive safe integer'
+      );
+    }
+
+    if (
+      !Number.isSafeInteger(this.windowSize) ||
+      this.windowSize < 1
+    ) {
+      throw new TypeError(
+        'backfill windowSize must be a positive safe integer'
+      );
+    }
+  }
+
+  async run({
+    startLedger,
+    endLedger,
+    filters = [],
+    pipeline = null,
+    onEvent,
+    onProgress,
+    signal,
+    checkpoint = null,
+    checkpointKey = 'default',
+    dedupe = true,
+    store = null
+  } = {}) {
+    validateLedger(startLedger, 'startLedger');
+    validateLedger(endLedger, 'endLedger');
+
+    if (endLedger < startLedger) {
+      throw new TypeError(
+        'endLedger must be greater than or equal to startLedger'
+      );
+    }
+
+    if (
+      pipeline != null &&
+      typeof pipeline.process !== 'function'
+    ) {
+      throw new TypeError(
+        'pipeline must implement process()'
+      );
+    }
+
+    if (typeof onEvent !== 'function') {
+      throw new TypeError(
+        'onEvent must be a function'
+      );
+    }
+
+    if (
+      onProgress != null &&
+      typeof onProgress !== 'function'
+    ) {
+      throw new TypeError(
+        'onProgress must be a function'
+      );
+    }
+
+    if (checkpoint != null) {
+      if (
+        typeof checkpoint.resumeFrom !== 'function' ||
+        typeof checkpoint.save !== 'function'
+      ) {
+        throw new TypeError(
+          'checkpoint must implement resumeFrom() and save()'
+        );
+      }
+    }
+
+    if (store != null) {
+      assertEventStore(store);
+    }
+
+    throwIfAborted(signal);
+
+    let effectiveStart = startLedger;
+
+    if (checkpoint) {
+      effectiveStart = await checkpoint.resumeFrom(
+        checkpointKey,
+        startLedger
+      );
+
+      if (
+        !Number.isSafeInteger(effectiveStart) ||
+        effectiveStart < 1
+      ) {
+        throw new BackfillError(
+          'checkpoint returned an invalid ledger'
+        );
+      }
+
+      if (effectiveStart > endLedger) {
+        return {
+          startLedger,
+          endLedger,
+          effectiveStart,
+          processed: 0,
+          windows: 0,
+          windowsCompleted: 0,
+          skipped: true
+        };
+      }
+    }
+
+    const windows = buildWindows(
+      effectiveStart,
+      endLedger,
+      this.windowSize
+    );
+
+    let processed = 0;
+    let windowsCompleted = 0;
+
+    const seen = dedupe ? new Set() : null;
+    const queue = windows.map((window, index) => ({
+      ...window,
+      index
+    }));
+
+    /*
+     * A window may finish before an earlier window.
+     *
+     * We therefore track completed window indexes separately
+     * and only advance the durable checkpoint across the
+     * contiguous completed prefix.
+     *
+     * Example:
+     *
+     *   window 0 = running
+     *   window 1 = complete
+     *
+     * checkpoint stays at window 0.
+     *
+     * Once window 0 completes, both can be committed:
+     *
+     *   checkpoint -> window 2 start
+     */
+
+    const completed = new Set();
+
+    let nextCheckpointIndex = 0;
+    let checkpointChain = Promise.resolve();
+
+    const advanceCheckpoint = () => {
+      if (!checkpoint) return checkpointChain;
+
+      checkpointChain = checkpointChain.then(async () => {
+        while (
+          completed.has(nextCheckpointIndex)
+        ) {
+          throwIfAborted(signal);
+
+          const window =
+            windows[nextCheckpointIndex];
+
+          await checkpoint.save(
+            window.endLedger + 1,
+            checkpointKey
+          );
+
+          completed.delete(nextCheckpointIndex);
+          nextCheckpointIndex++;
+        }
+      });
+
+      return checkpointChain;
+    };
+
+    const processWindow = async window => {
+      throwIfAborted(signal);
+
+      const events =
+        await this.streamer.getEventsWindowed({
+          startLedger: window.startLedger,
+          endLedger: window.endLedger,
+          filters,
+          limit: 10000,
+          signal
+        });
+
+      for (const event of events) {
+        throwIfAborted(signal);
+
+        if (
+          seen &&
+          event?.id != null
+        ) {
+          if (seen.has(event.id)) {
+            continue;
+          }
+
+          seen.add(event.id);
+        }
+
+        if (store != null) {
+          await store.put(event);
+        }
+
+        const processedEvent = pipeline
+          ? await pipeline.process(event, {
+              signal,
+              checkpointKey,
+              ledger: event.ledger,
+              window,
+              streamer: this.streamer
+            })
+          : event;
+
+        if (processedEvent === null) {
+          continue;
+        }
+
+        await onEvent(processedEvent);
+        processed++;
+      }
+
+      windowsCompleted++;
+
+      completed.add(window.index);
+
+      await advanceCheckpoint();
+
+      if (onProgress) {
+        await onProgress({
+          startLedger,
+          endLedger,
+          effectiveStart,
+          windowsTotal: windows.length,
+          windowsCompleted,
+          processed,
+          checkpointLedger:
+            checkpoint
+              ? nextCheckpointIndex < windows.length
+                ? windows[nextCheckpointIndex].startLedger
+                : endLedger + 1
+              : null,
+          currentWindow: {
+            startLedger: window.startLedger,
+            endLedger: window.endLedger
+          }
+        });
+      }
+    };
+
+    const worker = async () => {
+      while (true) {
+        throwIfAborted(signal);
+
+        const window = queue.shift();
+
+        if (!window) return;
+
+        await processWindow(window);
+      }
+    };
+
+    const workerCount = Math.min(
+      this.concurrency,
+      windows.length || 1
+    );
+
+    const workers = [];
+
+    for (let i = 0; i < workerCount; i++) {
+      workers.push(worker());
+    }
+
+    await Promise.all(workers);
+
+    /*
+     * All workers have finished. There may still be a queued
+     * checkpoint advancement waiting on the serialization chain.
+     */
+    await checkpointChain;
+
+    return {
+      startLedger,
+      endLedger,
+      effectiveStart,
+      processed,
+      windows: windows.length,
+      windowsCompleted,
+      skipped: false
+    };
+  }
+}
+
+function buildWindows(startLedger, endLedger, windowSize) {
+  const windows = [];
+
+  let cursor = startLedger;
+
+  while (cursor <= endLedger) {
+    const end = Math.min(
+      endLedger,
+      cursor + windowSize - 1
+    );
+
+    windows.push({
+      startLedger: cursor,
+      endLedger: end
+    });
+
+    cursor = end + 1;
+  }
+
+  return windows;
+}
+
+function validateLedger(value, name) {
+  if (
+    !Number.isSafeInteger(value) ||
+    value < 1
+  ) {
+    throw new TypeError(
+      `${name} must be a positive safe integer`
+    );
+  }
+}
+
+function throwIfAborted(signal) {
+  if (signal?.aborted) {
+    throw new BackfillError(
+      'backfill aborted',
+      {
+        cause: signal.reason
+      }
+    );
+  }
+}
+
+export { buildWindows };
+
+

package/src/checkpoint-file.js

@@ -0,0 +1,163 @@
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+
+import { CheckpointError, assertCheckpointStore } from './checkpoint.js';
+
+export class FileCheckpointStore {
+  constructor(directory) {
+    if (typeof directory !== 'string' || directory.length === 0) {
+      throw new TypeError(
+        'checkpoint directory must be a non-empty string'
+      );
+    }
+
+    this.directory = path.resolve(directory);
+
+    assertCheckpointStore(this);
+  }
+
+  async load(key) {
+    validateKey(key);
+
+    const file = this.#fileFor(key);
+
+    try {
+      const text = await fs.readFile(file, 'utf8');
+      const data = JSON.parse(text);
+
+      validateStoredCheckpoint(data);
+
+      return data.ledger;
+    } catch (error) {
+      if (error?.code === 'ENOENT') {
+        return null;
+      }
+
+      if (error instanceof CheckpointError) {
+        throw error;
+      }
+
+      if (error instanceof SyntaxError) {
+        throw new CheckpointError(
+          `invalid checkpoint file for key: ${key}`
+        );
+      }
+
+      throw error;
+    }
+  }
+
+  async save(key, ledger) {
+    validateKey(key);
+    validateLedger(ledger);
+
+    await fs.mkdir(this.directory, {
+      recursive: true
+    });
+
+    const file = this.#fileFor(key);
+    const temporary = `${file}.${process.pid}.${Date.now()}.tmp`;
+
+    const previous = await this.load(key);
+
+    if (previous != null && ledger < previous) {
+      throw new CheckpointError(
+        `checkpoint cannot move backwards: ${ledger} < ${previous}`
+      );
+    }
+
+    const data = {
+      version: 1,
+      key,
+      ledger
+    };
+
+    await fs.writeFile(
+      temporary,
+      `${JSON.stringify(data)}\n`,
+      'utf8'
+    );
+
+    await fs.rename(temporary, file);
+  }
+
+  async rewind(key, ledger) {
+    validateKey(key);
+    validateLedger(ledger);
+
+    await fs.mkdir(this.directory, {
+      recursive: true
+    });
+
+    const file = this.#fileFor(key);
+    const temporary =
+      `${file}.${process.pid}.${Date.now()}.tmp`;
+
+    const data = {
+      version: 1,
+      key,
+      ledger
+    };
+
+    await fs.writeFile(
+      temporary,
+      `${JSON.stringify(data)}\n`,
+      'utf8'
+    );
+
+    await fs.rename(
+      temporary,
+      file
+    );
+  }
+
+  async clear(key) {
+    validateKey(key);
+
+    try {
+      await fs.unlink(this.#fileFor(key));
+    } catch (error) {
+      if (error?.code !== 'ENOENT') {
+        throw error;
+      }
+    }
+  }
+
+  #fileFor(key) {
+    return path.join(
+      this.directory,
+      `${encodeURIComponent(key)}.json`
+    );
+  }
+}
+
+function validateKey(key) {
+  if (typeof key !== 'string' || key.length === 0) {
+    throw new TypeError(
+      'checkpoint key must be a non-empty string'
+    );
+  }
+}
+
+function validateLedger(ledger) {
+  if (!Number.isSafeInteger(ledger) || ledger < 1) {
+    throw new TypeError(
+      'checkpoint ledger must be a positive safe integer'
+    );
+  }
+}
+
+function validateStoredCheckpoint(data) {
+  if (
+    !data ||
+    data.version !== 1 ||
+    typeof data.key !== 'string'
+  ) {
+    throw new CheckpointError(
+      'invalid checkpoint file format'
+    );
+  }
+
+  validateKey(data.key);
+  validateLedger(data.ledger);
+}