webhook-delivery-engine 0.0.0-stage → 1.0.0

package/dist/src/storage/sqlite.adapter.js

@@ -0,0 +1,478 @@
+import { Buffer } from 'node:buffer';
+import Database from 'better-sqlite3';
+import { ATTEMPT_OUTCOMES, EVENT_STATUSES } from '../core/types.js';
+import { InvalidPageCursorError } from '../core/errors.js';
+import { runMigrations, } from './migrations/runner.js';
+const MAX_PAGE_SIZE = 1000;
+const ISO_TIMESTAMP_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/;
+function createMigrationConnection(database) {
+    return {
+        exec: (sql) => {
+            database.exec(sql);
+        },
+        prepare: (sql) => {
+            const statement = database.prepare(sql);
+            return {
+                all: (...parameters) => statement.all(...parameters),
+                get: (...parameters) => statement.get(...parameters),
+                run: (...parameters) => statement.run(...parameters),
+            };
+        },
+    };
+}
+/**
+ * SQLite implementation of the driver-neutral event persistence contract.
+ *
+ * Create instances with {@link SqliteDatabaseAdapter.open}; the returned
+ * adapter owns its database connection and must be closed by its owner.
+ */
+export class SqliteDatabaseAdapter {
+    database;
+    constructor(database) {
+        this.database = database;
+    }
+    /**
+     * Opens and initializes a SQLite database before exposing the adapter.
+     *
+     * Initialization enables WAL, a 5-second busy timeout, foreign-key checks,
+     * and applies the bundled forward-only migrations. The connection is closed
+     * and the original error is propagated if initialization fails.
+     *
+     * @param filename - SQLite database file path; use a file for WAL support.
+     * @returns An initialized adapter implementing {@link DatabaseAdapter}.
+     * @throws When the database cannot be opened, configured, or migrated.
+     */
+    static async open(filename) {
+        const database = new Database(filename);
+        try {
+            const journalMode = database.pragma('journal_mode = WAL', { simple: true });
+            if (journalMode !== 'wal') {
+                throw new Error('SQLite did not enable the required WAL journal mode.');
+            }
+            const busyTimeout = database.pragma('busy_timeout = 5000', { simple: true });
+            if (busyTimeout !== 5000) {
+                throw new Error('SQLite did not apply the required 5-second busy timeout.');
+            }
+            database.pragma('foreign_keys = ON');
+            const foreignKeys = database.pragma('foreign_keys', { simple: true });
+            if (foreignKeys !== 1) {
+                throw new Error('SQLite did not enable foreign-key enforcement.');
+            }
+            await runMigrations({
+                dialect: 'sqlite',
+                connection: createMigrationConnection(database),
+            });
+            return new SqliteDatabaseAdapter(database);
+        }
+        catch (error) {
+            database.close();
+            throw error;
+        }
+    }
+    /**
+     * Closes the SQLite connection owned by this adapter.
+     *
+     * @returns Nothing.
+     * @throws When the underlying database cannot be closed.
+     */
+    close() {
+        this.database.close();
+    }
+    /** @inheritdoc */
+    async upsertEvent(event) {
+        const createdAt = normalizeTimestamp(event.createdAt, 'createdAt');
+        const updatedAt = normalizeTimestamp(event.updatedAt, 'updatedAt');
+        const nextRetryAt = nullableTimestamp(event.nextRetryAt, 'nextRetryAt');
+        const lockedAt = nullableTimestamp(event.lockedAt, 'lockedAt');
+        const result = this.database
+            .prepare(`INSERT INTO webhook_events (
+          id, idempotency_key, client_id, destination_url, payload, metadata,
+          headers, status, attempt_count, next_retry_at, locked_at, locked_by,
+          created_at, updated_at
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        ON CONFLICT (client_id, idempotency_key) DO NOTHING`)
+            .run(event.id, event.idempotencyKey, event.clientId, event.destinationUrl, serializeJson(event.payload, 'payload'), serializeJsonObject(event.metadata, 'metadata'), serializeHeaders(event.headers), event.status, event.attemptCount, nextRetryAt, lockedAt, event.lockedBy, createdAt, updatedAt);
+        const row = this.database
+            .prepare('SELECT * FROM webhook_events WHERE client_id = ? AND idempotency_key = ?')
+            .get(event.clientId, event.idempotencyKey);
+        if (row === undefined) {
+            throw new Error('The event upsert completed without a matching event row.');
+        }
+        const storedEvent = readEvent(row);
+        return { event: storedEvent, inserted: result.changes === 1 };
+    }
+    /** @inheritdoc */
+    async claimDueEvents(input) {
+        assertPositiveSafeInteger(input.limit, 'limit');
+        const now = normalizeTimestamp(input.now, 'now');
+        const claimedEvents = this.database
+            .prepare(`UPDATE webhook_events
+         SET status = 'processing', locked_at = ?, locked_by = ?,
+             attempt_count = attempt_count + 1, updated_at = ?
+         WHERE id IN (
+           SELECT id FROM webhook_events
+           WHERE status IN ('pending', 'retry')
+             AND (next_retry_at IS NULL OR next_retry_at <= ?)
+           ORDER BY created_at ASC, id ASC
+           LIMIT ?
+         )
+         RETURNING *`)
+            .all(now, input.workerId, now, now, input.limit)
+            .map(readEvent);
+        return claimedEvents.sort(compareClaimedEvents);
+    }
+    /** @inheritdoc */
+    async recoverStaleLeases(input) {
+        const now = normalizeTimestamp(input.now, 'now');
+        if (!Number.isSafeInteger(input.timeoutLimitMs) || input.timeoutLimitMs <= 0) {
+            throw new RangeError('timeoutLimitMs must be a positive safe integer.');
+        }
+        const staleThresholdMs = Math.max(input.timeoutLimitMs + 30_000, 90_000);
+        if (!Number.isSafeInteger(staleThresholdMs)) {
+            throw new RangeError('timeoutLimitMs produces an unsafe stale lease threshold.');
+        }
+        const nowMilliseconds = Date.parse(now);
+        const staleBefore = normalizeTimestamp(new Date(nowMilliseconds - staleThresholdMs).toISOString(), 'staleBefore');
+        const nextRetryAt = normalizeTimestamp(new Date(nowMilliseconds + 10_000).toISOString(), 'nextRetryAt');
+        const result = this.database
+            .prepare(`UPDATE webhook_events
+         SET status = 'retry', next_retry_at = ?, locked_at = NULL,
+             locked_by = NULL, updated_at = ?
+         WHERE status = 'processing' AND locked_at < ?`)
+            .run(nextRetryAt, now, staleBefore);
+        return result.changes;
+    }
+    /** @inheritdoc */
+    async releaseEventLease(input) {
+        const releasedAt = normalizeTimestamp(input.releasedAt, 'releasedAt');
+        const result = this.database
+            .prepare(`UPDATE webhook_events
+         SET status = 'pending', next_retry_at = NULL, locked_at = NULL,
+             locked_by = NULL, attempt_count = attempt_count - 1, updated_at = ?
+         WHERE id = ? AND locked_by = ? AND status = 'processing'
+           AND attempt_count > 0`)
+            .run(releasedAt, input.eventId, input.workerId);
+        return result.changes === 1;
+    }
+    /** @inheritdoc */
+    async recordAttemptAndTransition(input) {
+        const { attempt, workerId, nextStatus, nextRetryAt, updatedAt } = input;
+        const attemptedAt = normalizeTimestamp(attempt.attemptedAt, 'attemptedAt');
+        const normalizedUpdatedAt = normalizeTimestamp(updatedAt, 'updatedAt');
+        const normalizedNextRetryAt = nullableTimestamp(nextRetryAt, 'nextRetryAt');
+        const transaction = this.database.transaction(() => {
+            const event = this.database
+                .prepare('SELECT attempt_count FROM webhook_events WHERE id = ? AND locked_by = ? AND status = ?')
+                .get(attempt.eventId, workerId, 'processing');
+            if (event === undefined)
+                return false;
+            const attemptCount = readNumberProperty(event, 'attempt_count');
+            if (attempt.attemptNumber !== attemptCount)
+                return false;
+            this.database
+                .prepare(`INSERT INTO webhook_event_attempts (
+            id, event_id, attempt_number, attempted_at, status_code,
+            response_snippet, error, latency_ms, outcome
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+                .run(attempt.id, attempt.eventId, attempt.attemptNumber, attemptedAt, attempt.statusCode, attempt.responseSnippet, attempt.error, attempt.latencyMs, attempt.outcome);
+            const result = this.database
+                .prepare(`UPDATE webhook_events
+           SET status = ?, next_retry_at = ?, locked_at = NULL, locked_by = NULL,
+               updated_at = ?
+           WHERE id = ? AND locked_by = ? AND status = 'processing'`)
+                .run(nextStatus, normalizedNextRetryAt, normalizedUpdatedAt, attempt.eventId, workerId);
+            if (result.changes !== 1) {
+                throw new Error('The event lease changed while recording the attempt.');
+            }
+            return true;
+        });
+        return transaction.immediate();
+    }
+    /** @inheritdoc */
+    async findEventById(eventId, clientId) {
+        const row = this.database
+            .prepare('SELECT * FROM webhook_events WHERE id = ? AND client_id = ?')
+            .get(eventId, clientId);
+        return row === undefined ? null : readEvent(row);
+    }
+    /** @inheritdoc */
+    async findEventByIdempotencyKey(clientId, idempotencyKey) {
+        const row = this.database
+            .prepare('SELECT * FROM webhook_events WHERE client_id = ? AND idempotency_key = ?')
+            .get(clientId, idempotencyKey);
+        return row === undefined ? null : readEvent(row);
+    }
+    /** @inheritdoc */
+    async listEvents(input) {
+        assertPositiveSafeInteger(input.limit, 'limit');
+        if (input.limit > MAX_PAGE_SIZE) {
+            throw new RangeError(`limit must not exceed ${MAX_PAGE_SIZE}.`);
+        }
+        const parameters = [input.clientId];
+        const filters = ['client_id = ?'];
+        if (input.status !== undefined) {
+            filters.push('status = ?');
+            parameters.push(input.status);
+        }
+        if (input.cursor !== undefined) {
+            const cursor = decodeCursor(input.cursor);
+            filters.push('(created_at > ? OR (created_at = ? AND id > ?))');
+            parameters.push(cursor.createdAt, cursor.createdAt, cursor.id);
+        }
+        const rows = this.database
+            .prepare(`SELECT * FROM webhook_events
+         WHERE ${filters.join(' AND ')}
+         ORDER BY created_at, id
+         LIMIT ?`)
+            .all(...parameters, input.limit + 1);
+        const hasNextPage = rows.length > input.limit;
+        const items = rows.slice(0, input.limit).map(readEvent);
+        const lastItem = items.at(-1);
+        return {
+            items,
+            nextCursor: hasNextPage && lastItem !== undefined ? encodeCursor(lastItem) : null,
+        };
+    }
+    /** @inheritdoc */
+    async listAttempts(eventId, clientId) {
+        const rows = this.database
+            .prepare(`SELECT attempts.* FROM webhook_event_attempts AS attempts
+         INNER JOIN webhook_events AS events ON events.id = attempts.event_id
+         WHERE events.id = ? AND events.client_id = ?
+         ORDER BY attempts.attempt_number`)
+            .all(eventId, clientId);
+        return rows.map(readAttempt);
+    }
+}
+/** Compares claimed events by creation time, then by their unique id. */
+export function compareClaimedEvents(left, right) {
+    if (left.createdAt !== right.createdAt) {
+        return left.createdAt < right.createdAt ? -1 : 1;
+    }
+    if (left.id < right.id)
+        return -1;
+    if (left.id > right.id)
+        return 1;
+    return 0;
+}
+function readEvent(value) {
+    const row = asRow(value, 'event');
+    const status = readStringProperty(row, 'status');
+    if (!EVENT_STATUSES.includes(status)) {
+        throw new Error('The database contains an invalid event status.');
+    }
+    const payload = parseJson(readStringProperty(row, 'payload'), 'payload');
+    const metadata = parseJsonObject(readStringProperty(row, 'metadata'), 'metadata');
+    const headers = parseHeaders(readStringProperty(row, 'headers'));
+    return {
+        id: readStringProperty(row, 'id'),
+        idempotencyKey: readStringProperty(row, 'idempotency_key'),
+        clientId: readStringProperty(row, 'client_id'),
+        destinationUrl: readStringProperty(row, 'destination_url'),
+        payload,
+        metadata,
+        headers,
+        status: status,
+        attemptCount: readNumberProperty(row, 'attempt_count'),
+        nextRetryAt: readNullableTimestamp(row, 'next_retry_at'),
+        lockedAt: readNullableTimestamp(row, 'locked_at'),
+        lockedBy: readNullableStringProperty(row, 'locked_by'),
+        createdAt: normalizeTimestamp(readStringProperty(row, 'created_at'), 'created_at'),
+        updatedAt: normalizeTimestamp(readStringProperty(row, 'updated_at'), 'updated_at'),
+    };
+}
+function readAttempt(value) {
+    const row = asRow(value, 'attempt');
+    const outcome = readStringProperty(row, 'outcome');
+    if (!ATTEMPT_OUTCOMES.includes(outcome)) {
+        throw new Error('The database contains an invalid attempt outcome.');
+    }
+    return {
+        id: readStringProperty(row, 'id'),
+        eventId: readStringProperty(row, 'event_id'),
+        attemptNumber: readNumberProperty(row, 'attempt_number'),
+        attemptedAt: normalizeTimestamp(readStringProperty(row, 'attempted_at'), 'attempted_at'),
+        statusCode: readNullableNumberProperty(row, 'status_code'),
+        responseSnippet: readNullableStringProperty(row, 'response_snippet'),
+        error: readNullableStringProperty(row, 'error'),
+        latencyMs: readNullableNumberProperty(row, 'latency_ms'),
+        outcome: outcome,
+    };
+}
+function asRow(value, description) {
+    if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+        throw new Error(`The database returned an invalid ${description} row.`);
+    }
+    return value;
+}
+function readStringProperty(row, key) {
+    const value = asRow(row, 'database')[key];
+    if (typeof value !== 'string') {
+        throw new Error(`The database returned an invalid "${key}" value.`);
+    }
+    return value;
+}
+function readNullableStringProperty(row, key) {
+    const value = row[key];
+    if (value === null)
+        return null;
+    if (typeof value !== 'string') {
+        throw new Error(`The database returned an invalid "${key}" value.`);
+    }
+    return value;
+}
+function readNumberProperty(row, key) {
+    const value = asRow(row, 'database')[key];
+    if (typeof value !== 'number' || !Number.isSafeInteger(value)) {
+        throw new Error(`The database returned an invalid "${key}" value.`);
+    }
+    return value;
+}
+function readNullableNumberProperty(row, key) {
+    const value = row[key];
+    if (value === null)
+        return null;
+    if (typeof value !== 'number' || !Number.isSafeInteger(value)) {
+        throw new Error(`The database returned an invalid "${key}" value.`);
+    }
+    return value;
+}
+function readNullableTimestamp(row, key) {
+    const value = readNullableStringProperty(row, key);
+    return value === null ? null : normalizeTimestamp(value, key);
+}
+function normalizeTimestamp(value, field) {
+    const milliseconds = Date.parse(value);
+    if (!ISO_TIMESTAMP_PATTERN.test(value) || !Number.isFinite(milliseconds)) {
+        throw new TypeError(`${field} must be a valid ISO-8601 timestamp.`);
+    }
+    return new Date(milliseconds).toISOString();
+}
+function nullableTimestamp(value, field) {
+    return value === null ? null : normalizeTimestamp(value, field);
+}
+function serializeJson(value, field) {
+    if (!isJsonValue(value)) {
+        throw new TypeError(`${field} must contain JSON-compatible values.`);
+    }
+    const serialized = JSON.stringify(value);
+    if (serialized === undefined) {
+        throw new TypeError(`${field} could not be serialized as JSON.`);
+    }
+    return serialized;
+}
+function serializeJsonObject(value, field) {
+    if (!isJsonObject(value)) {
+        throw new TypeError(`${field} must be a JSON object.`);
+    }
+    return serializeJson(value, field);
+}
+function serializeHeaders(headers) {
+    if (!isStringRecord(headers)) {
+        throw new TypeError('headers must contain only string values.');
+    }
+    return JSON.stringify(headers);
+}
+function parseJson(value, field) {
+    let parsed;
+    try {
+        parsed = JSON.parse(value);
+    }
+    catch (cause) {
+        throw new TypeError(`The persisted ${field} value is invalid JSON.`, { cause });
+    }
+    if (!isJsonValue(parsed)) {
+        throw new TypeError(`The persisted ${field} value is not JSON-compatible.`);
+    }
+    return parsed;
+}
+function parseJsonObject(value, field) {
+    const parsed = parseJson(value, field);
+    if (!isJsonObject(parsed)) {
+        throw new TypeError(`The persisted ${field} value must be a JSON object.`);
+    }
+    return parsed;
+}
+function parseHeaders(value) {
+    let parsed;
+    try {
+        parsed = JSON.parse(value);
+    }
+    catch (cause) {
+        throw new TypeError('The persisted headers value is invalid JSON.', { cause });
+    }
+    if (!isStringRecord(parsed)) {
+        throw new TypeError('The persisted headers value must be a string object.');
+    }
+    return parsed;
+}
+function isJsonValue(value) {
+    return validateJsonValue(value, new WeakSet());
+}
+function validateJsonValue(value, ancestors) {
+    if (value === null || typeof value === 'string' || typeof value === 'boolean')
+        return true;
+    if (typeof value === 'number')
+        return Number.isFinite(value);
+    if (typeof value !== 'object' || value === null)
+        return false;
+    if (ancestors.has(value))
+        return false;
+    ancestors.add(value);
+    const valid = Array.isArray(value)
+        ? value.every((entry) => validateJsonValue(entry, ancestors))
+        : isPlainJsonObject(value) &&
+            Object.values(value).every((entry) => validateJsonValue(entry, ancestors));
+    ancestors.delete(value);
+    return valid;
+}
+function isJsonObject(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        !Array.isArray(value) &&
+        isJsonValue(value) &&
+        isPlainJsonObject(value));
+}
+function isPlainJsonObject(value) {
+    if (Array.isArray(value))
+        return false;
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null;
+}
+function isStringRecord(value) {
+    if (typeof value !== 'object' || value === null || Array.isArray(value))
+        return false;
+    const prototype = Object.getPrototypeOf(value);
+    if (prototype !== Object.prototype && prototype !== null)
+        return false;
+    return Object.values(value).every((entry) => typeof entry === 'string');
+}
+function assertPositiveSafeInteger(value, field) {
+    if (!Number.isSafeInteger(value) || value <= 0) {
+        throw new RangeError(`${field} must be a positive safe integer.`);
+    }
+}
+function encodeCursor(event) {
+    return Buffer.from(JSON.stringify([event.createdAt, event.id]), 'utf8').toString('base64url');
+}
+function decodeCursor(cursor) {
+    let decoded;
+    try {
+        decoded = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
+    }
+    catch (cause) {
+        throw new InvalidPageCursorError({ cause });
+    }
+    if (!Array.isArray(decoded) ||
+        decoded.length !== 2 ||
+        typeof decoded[0] !== 'string' ||
+        typeof decoded[1] !== 'string') {
+        throw new InvalidPageCursorError();
+    }
+    try {
+        return { createdAt: normalizeTimestamp(decoded[0], 'cursor'), id: decoded[1] };
+    }
+    catch (cause) {
+        throw new InvalidPageCursorError({ cause });
+    }
+}

package/package.json

@@ -1,6 +1,78 @@
 {
   "name": "webhook-delivery-engine",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.0.0",
+  "description": "Embeddable Node.js webhook delivery engine with persistent retries.",
+  "license": "MIT",
+  "repository": "github:hamidjavadi/webhook-delivery-engine",
+  "keywords": [
+    "webhook",
+    "webhook-delivery",
+    "webhook-retry",
+    "delivery",
+    "event-delivery",
+    "retry",
+    "idempotency",
+    "queue",
+    "sqlite",
+    "nodejs",
+    "typescript"
+  ],
+  "type": "module",
+  "main": "./dist/src/index.js",
+  "types": "./dist/src/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/src/index.js"
+    }
+  },
+  "files": [
+    "dist/src"
+  ],
+  "scripts": {
+    "build": "tsc --project tsconfig.json && node --input-type=module -e \"import { cpSync } from 'node:fs'; cpSync('src/storage/migrations/sql', 'dist/src/storage/migrations/sql', { recursive: true });\"",
+    "test": "vitest run",
+    "benchmark:ingestion": "tsx scripts/benchmark-ingestion.ts",
+    "test:watch": "vitest",
+    "lint": "eslint . --max-warnings 0",
+    "format": "prettier --write ."
+  },
+  "dependencies": {
+    "better-sqlite3": "^11.8.1",
+    "dotenv": "^16.4.7",
+    "pino": "^9.6.0",
+    "undici": "8.12.3",
+    "zod": "^3.24.2"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.20.0",
+    "@types/better-sqlite3": "^7.6.12",
+    "@types/node": "^24.0.0",
+    "@vitest/coverage-v8": "^3.2.7",
+    "eslint": "^9.20.1",
+    "node-gyp": "^13.1.0",
+    "prettier": "^3.5.1",
+    "tsx": "^4.19.3",
+    "typescript": "^5.7.3",
+    "typescript-eslint": "^8.24.0",
+    "vitest": "^3.0.5"
+  },
+  "lint-staged": {
+    "*.ts": [
+      "eslint --fix",
+      "prettier --write"
+    ],
+    "*.{json,md}": [
+      "prettier --write"
+    ]
+  },
+  "packageManager": "pnpm@10.4.1",
+  "pnpm": {
+    "onlyBuiltDependencies": [
+      "better-sqlite3"
+    ]
+  },
+  "engines": {
+    "node": ">=24.0.0"
+  }
+}