soroban-events 0.1.0

package/README.md

@@ -0,0 +1,230 @@
+# soroban-events
+
+Lightweight Soroban RPC event ingestion, pagination, retries, deduplication, and ScVal decoding for Node.js.
+
+Build Soroban dashboards, activity feeds, lightweight indexers, analytics tools, monitoring services, bots, and event-driven backends without implementing RPC pagination and event decoding yourself.
+
+## Features
+
+- Ledger-range windowing
+- RPC cursor pagination
+- Event deduplication
+- Rate-limit and transient retries
+- Soroban `ScVal` decoding
+- Large integer precision preservation
+- Contract event filtering
+- Recent-event lookup with `tail()`
+- Async event streaming with `stream()`
+- AbortSignal support
+
+## Install
+
+```bash
+npm install soroban-events
+```
+
+Requires Node.js 20+.
+
+## Quick start
+
+```js
+import { SorobanEventStreamer } from 'soroban-events';
+
+const streamer = new SorobanEventStreamer(
+  'https://soroban-testnet.stellar.org'
+);
+
+const latest = await streamer.getLatestLedger();
+
+const events = await streamer.getEventsWindowed({
+  startLedger: latest - 100,
+  endLedger: latest,
+  filters: [{ type: 'contract' }],
+  limit: 10
+});
+
+for (const event of events) {
+  console.log(event);
+}
+```
+
+## Filter by contract
+
+```js
+const events = await streamer.getEventsWindowed({
+  startLedger: latest - 1000,
+  endLedger: latest,
+  filters: [
+    {
+      type: 'contract',
+      contractIds: ['YOUR_CONTRACT_ID']
+    }
+  ],
+  limit: 100
+});
+```
+
+## Get recent events
+
+```js
+const events = await streamer.tail({
+  contractId: 'YOUR_CONTRACT_ID',
+  limit: 20
+});
+
+console.log(events);
+```
+
+## Stream events
+
+```js
+for await (const event of streamer.stream({
+  startLedger: 4750000,
+  filters: [
+    {
+      type: 'contract',
+      contractIds: ['YOUR_CONTRACT_ID']
+    }
+  ]
+})) {
+  console.log('New event:', event);
+}
+```
+
+Stop a stream with `AbortController`:
+
+```js
+const controller = new AbortController();
+
+setTimeout(() => controller.abort(), 30_000);
+
+for await (const event of streamer.stream({
+  startLedger: 4750000,
+  signal: controller.signal
+})) {
+  console.log(event);
+}
+```
+
+## Decode ScVal
+
+The decoder can also be used independently:
+
+```js
+import { unwrapScVal } from 'soroban-events';
+
+const value = unwrapScVal(scVal);
+console.log(value);
+```
+
+Common Soroban values supported include integers, symbols, strings, booleans, bytes, addresses, vectors, and maps.
+
+Large integer values are normalized without relying on JavaScript `Number` precision.
+
+## Event format
+
+Events are normalized into a consistent object containing fields such as:
+
+```js
+{
+  id,
+  type,
+  ledger,
+  ledgerClosedAt,
+  contractId,
+  transactionIndex,
+  operationIndex,
+  txHash,
+  topics,
+  value,
+  inSuccessfulContractCall
+}
+```
+
+## Configuration
+
+```js
+const streamer = new SorobanEventStreamer(RPC_URL, {
+  pollInterval: 3000,
+  windowSize: 9500,
+  pageSize: 1000,
+  maxRetries: 3,
+  retryBaseMs: 500
+});
+```
+
+| Option | Default | Description |
+|---|---:|---|
+| `pollInterval` | `3000` | Polling interval for `stream()` |
+| `windowSize` | `9500` | Maximum ledger window |
+| `pageSize` | `1000` | RPC page size |
+| `maxRetries` | `3` | Maximum retry attempts |
+| `retryBaseMs` | `500` | Base retry delay |
+
+## How it works
+
+```text
+Soroban RPC
+    |
+    v
+ledger windows
+    |
+    v
+cursor pagination
+    |
+    v
+retry handling
+    |
+    v
+deduplication
+    |
+    v
+ScVal decoding
+    |
+    v
+your application
+```
+
+No database is required.
+
+No full indexer stack is required.
+
+Use the normalized events in whatever application or storage layer you need.
+
+## Testing
+
+Run the unit tests:
+
+```bash
+npm test
+```
+
+Live Stellar Testnet verification:
+
+```bash
+npm run test:live
+```
+
+The project currently includes 22 automated tests covering windowing, pagination, deduplication, limits, retries, abort handling, filtering, ScVal decoding, large integers, and event normalization.
+
+The live test has also been verified against Stellar Testnet RPC.
+
+## Status
+
+**v0.1.0**
+
+The current release focuses on the core RPC ingestion and ScVal decoding layer.
+
+This is intentionally a lightweight library, not a database-backed blockchain indexer.
+
+Potential future work includes durable cursor persistence, stronger recovery strategies, webhook delivery, richer filtering helpers, and production indexing integrations.
+
+## Contributing
+
+Bug reports, edge cases, documentation improvements, test cases, and pull requests are welcome.
+
+When reporting an RPC issue, include the endpoint/network, ledger range, relevant RPC error or response, expected behavior, and actual behavior.
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "soroban-events",
+  "version": "0.1.0",
+  "description": "Resilient, windowed event streamer and XDR decoder for Soroban RPC",
+  "type": "module",
+  "main": "./src/index.js",
+  "exports": {
+    ".": "./src/index.js"
+  },
+  "files": [
+    "src/"
+  ],
+  "scripts": {
+    "test": "node --test tests/*.test.js",
+    "test:live": "node tests/test-engine.js",
+    "pack:check": "npm pack --dry-run"
+  },
+  "keywords": [
+    "soroban",
+    "stellar",
+    "events",
+    "event-streaming",
+    "event-indexer",
+    "xdr",
+    "rpc"
+  ],
+  "license": "MIT",
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "@stellar/stellar-sdk": "^17.1.0"
+  }
+}

package/src/decoder.js

@@ -0,0 +1,142 @@
+import { scValToNative } from '@stellar/stellar-sdk';
+
+const DEFAULT_MAX_DEPTH = 50;
+
+export function unwrapScVal(scVal, options = {}) {
+  if (scVal == null) return null;
+
+  const maxDepth = options.maxDepth ?? DEFAULT_MAX_DEPTH;
+
+  try {
+    return normalizeNative(scValToNative(scVal), 0, maxDepth);
+  } catch {
+    return parseScValDirect(scVal);
+  }
+}
+
+function normalizeNative(value, depth, maxDepth) {
+  if (depth > maxDepth) {
+    throw new Error(`ScVal nesting exceeds maximum depth of ${maxDepth}`);
+  }
+
+  if (value === null || value === undefined) return null;
+
+  if (typeof value === 'bigint') return value.toString();
+
+  if (value instanceof Uint8Array || Buffer.isBuffer(value)) {
+    return Buffer.from(value).toString('hex');
+  }
+
+  if (Array.isArray(value)) {
+    return value.map(v => normalizeNative(v, depth + 1, maxDepth));
+  }
+
+  if (value instanceof Map) {
+    return new Map(
+      [...value.entries()].map(([k, v]) => [
+        normalizeNative(k, depth + 1, maxDepth),
+        normalizeNative(v, depth + 1, maxDepth)
+      ])
+    );
+  }
+
+  if (typeof value === 'object') {
+    const result = Object.create(null);
+
+    for (const [key, val] of Object.entries(value)) {
+      result[key] = normalizeNative(val, depth + 1, maxDepth);
+    }
+
+    return result;
+  }
+
+  return value;
+}
+
+function parseScValDirect(scVal) {
+  try {
+    if (!scVal?.switch) return null;
+
+    const name = scVal.switch().name;
+
+    switch (name) {
+      case 'scvVoid': return null;
+      case 'scvBool': return scVal.b();
+      case 'scvU32': return scVal.u32();
+      case 'scvI32': return scVal.i32();
+      case 'scvU64': return scVal.u64().toString();
+      case 'scvI64': return scVal.i64().toString();
+      case 'scvU128': return scVal.u128().toString();
+      case 'scvI128': return scVal.i128().toString();
+      case 'scvU256': return scVal.u256().toString();
+      case 'scvI256': return scVal.i256().toString();
+      case 'scvSymbol': return scVal.sym().toString();
+      case 'scvString': return scVal.str().toString();
+      case 'scvBytes':
+        return Buffer.from(
+          scVal.bytes?.() ?? scVal.bin?.() ?? []
+        ).toString('hex');
+      case 'scvAddress':
+        return scVal.address().toString();
+      default:
+        return `[Unresolved ScVal: ${name}]`;
+    }
+  } catch (error) {
+    return `[ScVal decode error: ${error.message}]`;
+  }
+}
+
+function normalizeContractId(value) {
+  if (value == null || typeof value === 'string') return value;
+
+  if (typeof value.toString === 'function') {
+    const text = value.toString();
+
+    if (
+      text &&
+      text !== '[object Object]' &&
+      !/^\[object Object\]$/.test(text)
+    ) {
+      return text;
+    }
+  }
+
+  return value;
+}
+
+export function decodeEvent(rawEvent, options = {}) {
+  const topics = Array.isArray(rawEvent.topic)
+    ? rawEvent.topic.map(topic => {
+        try {
+          return unwrapScVal(topic, options);
+        } catch {
+          return topic;
+        }
+      })
+    : [];
+
+  let value = null;
+
+  if (rawEvent.value != null) {
+    try {
+      value = unwrapScVal(rawEvent.value, options);
+    } catch {
+      value = rawEvent.value;
+    }
+  }
+
+  return {
+    id: rawEvent.id,
+    type: rawEvent.type,
+    ledger: rawEvent.ledger,
+    ledgerClosedAt: rawEvent.ledgerClosedAt,
+    contractId: normalizeContractId(rawEvent.contractId),
+    transactionIndex: rawEvent.transactionIndex,
+    operationIndex: rawEvent.operationIndex,
+    txHash: rawEvent.txHash,
+    topics,
+    value,
+    inSuccessfulContractCall:
+      rawEvent.inSuccessfulContractCall ?? true
+  };
+}

package/src/index.js

@@ -0,0 +1,10 @@
+export {
+  SorobanEventStreamer,
+  MAX_SAFE_LEDGER_SPAN,
+  DEFAULT_PAGE_SIZE
+} from './streamer.js';
+
+export {
+  unwrapScVal,
+  decodeEvent
+} from './decoder.js';

package/src/streamer.js

@@ -0,0 +1,312 @@
+import { rpc } from '@stellar/stellar-sdk';
+import { decodeEvent } from './decoder.js';
+
+export const MAX_SAFE_LEDGER_SPAN = 9500;
+export const DEFAULT_PAGE_SIZE = 1000;
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function classifyError(error) {
+  const message = String(error?.message ?? error).toLowerCase();
+
+  const status =
+    error?.response?.status ??
+    error?.status ??
+    error?.statusCode;
+
+  if (status === 429 || message.includes('429') || message.includes('rate limit')) {
+    return 'rate-limit';
+  }
+
+  if (
+    status >= 500 ||
+    message.includes('timeout') ||
+    message.includes('timed out') ||
+    message.includes('econnreset') ||
+    message.includes('socket')
+  ) {
+    return 'transient';
+  }
+
+  return 'fatal';
+}
+
+export class SorobanEventStreamer {
+  constructor(rpcUrl, options = {}) {
+    if (!rpcUrl) {
+      throw new TypeError('rpcUrl is required');
+    }
+
+    this.server = new rpc.Server(
+      rpcUrl,
+      options.serverOptions || {}
+    );
+
+    this.pollInterval = options.pollInterval ?? 3000;
+    this.windowSize = Math.min(
+      options.windowSize ?? MAX_SAFE_LEDGER_SPAN,
+      MAX_SAFE_LEDGER_SPAN
+    );
+
+    this.pageSize = Math.min(
+      options.pageSize ?? DEFAULT_PAGE_SIZE,
+      10000
+    );
+
+    this.maxRetries = options.maxRetries ?? 3;
+    this.retryBaseMs = options.retryBaseMs ?? 500;
+  }
+
+  async getLatestLedger() {
+    const response = await this.server.getLatestLedger();
+    return response.sequence;
+  }
+
+  async getEventsWindowed({
+    startLedger,
+    endLedger,
+    filters = [],
+    limit = 100,
+    signal
+  }) {
+    if (!Number.isInteger(startLedger) || startLedger < 1) {
+      throw new TypeError('startLedger must be a positive integer');
+    }
+
+    if (endLedger != null && (!Number.isInteger(endLedger) || endLedger < startLedger)) {
+      throw new TypeError('endLedger must be >= startLedger');
+    }
+
+    const targetEnd = endLedger ?? await this.getLatestLedger();
+
+    if (signal?.aborted) {
+      throw new DOMException('Operation aborted', 'AbortError');
+    }
+
+    const results = [];
+    const seen = new Set();
+
+    let currentStart = startLedger;
+
+    while (currentStart <= targetEnd && results.length < limit) {
+      if (signal?.aborted) {
+        throw new DOMException('Operation aborted', 'AbortError');
+      }
+
+      const currentEndExclusive = Math.min(
+        currentStart + this.windowSize,
+        targetEnd + 1
+      );
+
+      const remaining = limit - results.length;
+
+      const events = await this.fetchWindow(
+        currentStart,
+        currentEndExclusive,
+        filters,
+        remaining,
+        signal
+      );
+
+      for (const raw of events) {
+        if (!raw?.id || seen.has(raw.id)) continue;
+
+        seen.add(raw.id);
+        results.push(decodeEvent(raw));
+
+        if (results.length >= limit) break;
+      }
+
+      currentStart = currentEndExclusive;
+    }
+
+    return results;
+  }
+
+  async fetchWindow(
+    startLedger,
+    endLedgerExclusive,
+    filters,
+    limit,
+    signal
+  ) {
+    const rawEvents = [];
+    let cursor;
+
+    while (rawEvents.length < limit) {
+      if (signal?.aborted) {
+        throw new DOMException('Operation aborted', 'AbortError');
+      }
+
+      const pagination = {
+        limit: Math.min(this.pageSize, limit - rawEvents.length)
+      };
+
+      if (cursor) {
+        pagination.cursor = cursor;
+      }
+
+      const params = {
+        filters,
+        pagination
+      };
+
+      if (!cursor) {
+        params.startLedger = startLedger;
+        params.endLedger = endLedgerExclusive;
+      }
+
+      const response = await this.requestWithRetry(params, signal);
+
+      const events = response?.events ?? [];
+
+      rawEvents.push(...events);
+
+      const nextCursor = response?.cursor;
+
+      if (!nextCursor || events.length === 0) {
+        break;
+      }
+
+      if (nextCursor === cursor) {
+        throw new Error(
+          `RPC pagination cursor did not advance for ledger range ${startLedger}-${endLedgerExclusive}`
+        );
+      }
+
+      cursor = nextCursor;
+    }
+
+    return rawEvents.slice(0, limit);
+  }
+
+  async requestWithRetry(params, signal) {
+    let attempt = 0;
+
+    while (true) {
+      if (signal?.aborted) {
+        throw new DOMException('Operation aborted', 'AbortError');
+      }
+
+      try {
+        return await this.server.getEvents(params);
+      } catch (error) {
+        const kind = classifyError(error);
+
+        if (
+          (kind !== 'rate-limit' && kind !== 'transient') ||
+          attempt >= this.maxRetries
+        ) {
+          throw error;
+        }
+
+        const delay =
+          this.retryBaseMs *
+          2 ** attempt *
+          (kind === 'rate-limit' ? 2 : 1);
+
+        await sleep(delay);
+        attempt++;
+      }
+    }
+  }
+
+  async tail({
+    contractId,
+    limit = 10,
+    maxLookbackLedgers = 50000,
+    filters
+  } = {}) {
+    if (limit <= 0) return [];
+
+    const latest = await this.getLatestLedger();
+
+    const effectiveFilters =
+      filters ??
+      (contractId
+        ? [{ type: 'contract', contractIds: [contractId] }]
+        : [{ type: 'contract' }]);
+
+    const collected = [];
+    const seen = new Set();
+
+    let endExclusive = latest + 1;
+    const minimumLedger = Math.max(
+      1,
+      latest - maxLookbackLedgers
+    );
+
+    while (
+      endExclusive > minimumLedger &&
+      collected.length < limit
+    ) {
+      const start = Math.max(
+        minimumLedger,
+        endExclusive - this.windowSize
+      );
+
+      const events = await this.fetchWindow(
+        start,
+        endExclusive,
+        effectiveFilters,
+        this.pageSize
+      );
+
+      for (let i = events.length - 1; i >= 0; i--) {
+        const event = events[i];
+
+        if (!event?.id || seen.has(event.id)) continue;
+
+        seen.add(event.id);
+        collected.unshift(decodeEvent(event));
+
+        if (collected.length >= limit) break;
+      }
+
+      endExclusive = start;
+    }
+
+    return collected.slice(-limit);
+  }
+
+  async *stream({
+    startLedger,
+    filters = [],
+    pollInterval = this.pollInterval,
+    signal
+  } = {}) {
+    let cursorLedger =
+      startLedger ?? await this.getLatestLedger();
+
+    while (!signal?.aborted) {
+      const latest = await this.getLatestLedger();
+
+      if (cursorLedger <= latest) {
+        const events = await this.getEventsWindowed({
+          startLedger: cursorLedger,
+          endLedger: latest,
+          filters,
+          limit: 10000,
+          signal
+        });
+
+        for (const event of events) {
+          if (signal?.aborted) return;
+
+          yield event;
+
+          cursorLedger = Math.max(
+            cursorLedger,
+            event.ledger + 1
+          );
+        }
+      }
+
+      if (cursorLedger > latest) {
+        await sleep(pollInterval);
+      }
+    }
+  }
+}