queue-tool 1.0.0

package/dist/adapters/redis.test.js

@@ -0,0 +1,102 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const redis_1 = require("./redis");
+const ioredis_1 = __importDefault(require("ioredis"));
+describe('RedisQueueAdapter Integration Tests', () => {
+    let adapter;
+    let redisClient;
+    const queueName = 'test-redis-integration-queue';
+    const connectionString = 'redis://localhost:6379';
+    let isAvailable = true;
+    beforeAll(async () => {
+        try {
+            redisClient = new ioredis_1.default(connectionString);
+            await redisClient.ping();
+            adapter = new redis_1.RedisQueueAdapter(connectionString);
+            await adapter.init();
+        }
+        catch (err) {
+            isAvailable = false;
+            console.warn('Redis is not running. Skipping Redis integration tests.');
+        }
+    });
+    afterAll(async () => {
+        if (!isAvailable)
+            return;
+        await adapter.close();
+        await redisClient.quit();
+    });
+    beforeEach(async () => {
+        if (!isAvailable)
+            return;
+        // Clean keys related to our test queue
+        const keys = await redisClient.keys(`*${queueName}*`);
+        if (keys.length > 0) {
+            await redisClient.del(...keys);
+        }
+    });
+    test('should enqueue a job successfully', async () => {
+        if (!isAvailable)
+            return;
+        const payload = { task: 'test-node-enqueue', value: 100 };
+        const job = await adapter.enqueue(queueName, payload, { priority: 10 });
+        expect(job.id).toBeDefined();
+        expect(job.queue).toBe(queueName);
+        expect(job.payload).toEqual(payload);
+        expect(job.status).toBe('pending');
+        expect(job.priority).toBe(10);
+    });
+    test('should fail validation on invalid priority bounds', async () => {
+        if (!isAvailable)
+            return;
+        const payload = { task: 'test-priority-bounds' };
+        await expect(adapter.enqueue(queueName, payload, { priority: 2000 })).rejects.toThrow('Priority must be in the range [-1000, 1000]');
+    });
+    test('should claim and complete a job successfully', async () => {
+        if (!isAvailable)
+            return;
+        const payload = { task: 'test-node-claim', value: 200 };
+        const job = await adapter.enqueue(queueName, payload);
+        const claimedJob = await adapter.claimJob(queueName, 'worker-test-node-1');
+        expect(claimedJob).not.toBeNull();
+        expect(claimedJob.id).toBe(job.id);
+        expect(claimedJob.status).toBe('processing');
+        expect(claimedJob.attempts).toBe(1);
+        await adapter.completeJob(claimedJob.id);
+        const finalJob = await adapter.getJob(job.id);
+        expect(finalJob).not.toBeNull();
+        expect(finalJob.status).toBe('completed');
+    });
+    test('should fail and retry job within maxAttempts limit', async () => {
+        if (!isAvailable)
+            return;
+        const payload = { task: 'test-node-retry' };
+        const job = await adapter.enqueue(queueName, payload, { maxAttempts: 2 });
+        const claimedJob = await adapter.claimJob(queueName, 'worker-test-node-2');
+        expect(claimedJob).not.toBeNull();
+        await adapter.failJob(claimedJob.id, 'First test error');
+        const retriedJob = await adapter.getJob(job.id);
+        expect(retriedJob).not.toBeNull();
+        expect(retriedJob.status).toBe('pending');
+        expect(retriedJob.error).toBe('First test error');
+        // Manually update run_at to past to bypass 5s retry delay
+        const jobKey = `job:${job.id}`;
+        const pastTime = Date.now() - 60000;
+        await redisClient.hset(jobKey, 'run_at', pastTime.toString());
+        const score = pastTime - (job.priority * 10000000000);
+        await redisClient.zadd(`queue:${queueName}:pending`, score, job.id);
+        // Claim again (2nd attempt)
+        const claimedJob2 = await adapter.claimJob(queueName, 'worker-test-node-2');
+        expect(claimedJob2).not.toBeNull();
+        expect(claimedJob2.attempts).toBe(2);
+        // Fail again -> terminal failure
+        await adapter.failJob(claimedJob2.id, 'Second test error');
+        const finalJob = await adapter.getJob(job.id);
+        expect(finalJob).not.toBeNull();
+        expect(finalJob.status).toBe('failed');
+        expect(finalJob.error).toBe('Second test error');
+    });
+});

package/dist/factory.d.ts

@@ -0,0 +1,4 @@
+import { IQueueAdapter } from './types';
+export declare class QueueFactory {
+    static create(connectionString: string): Promise<IQueueAdapter>;
+}

package/dist/factory.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QueueFactory = void 0;
+const postgres_1 = require("./adapters/postgres");
+const redis_1 = require("./adapters/redis");
+const mongodb_1 = require("./adapters/mongodb");
+const logger_1 = require("./utils/logger");
+class QueueFactory {
+    static async create(connectionString) {
+        logger_1.logger.info('Creating queue adapter...', { connectionString });
+        if (connectionString.startsWith('postgres://') || connectionString.startsWith('postgresql://')) {
+            logger_1.logger.info('Selected Postgres Queue Adapter');
+            const adapter = new postgres_1.PostgresQueueAdapter(connectionString);
+            await adapter.init();
+            return adapter;
+        }
+        else if (connectionString.startsWith('redis://') || connectionString.startsWith('rediss://')) {
+            logger_1.logger.info('Selected Redis Queue Adapter');
+            const adapter = new redis_1.RedisQueueAdapter(connectionString);
+            await adapter.init();
+            return adapter;
+        }
+        else if (connectionString.startsWith('mongodb://') || connectionString.startsWith('mongodb+srv://')) {
+            logger_1.logger.info('Selected MongoDB Queue Adapter');
+            const adapter = new mongodb_1.MongoQueueAdapter(connectionString);
+            await adapter.init();
+            return adapter;
+        }
+        else {
+            logger_1.logger.error('Unsupported connection string', { connectionString });
+            throw new Error(`Unsupported connection string: ${connectionString}`);
+        }
+    }
+}
+exports.QueueFactory = QueueFactory;

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export * from './types';
+export { QueueFactory } from './factory';
+export { PostgresQueueAdapter } from './adapters/postgres';
+export { RedisQueueAdapter } from './adapters/redis';
+export { MongoQueueAdapter } from './adapters/mongodb';
+export { Worker, WorkerOptions } from './worker';
+export { serialize, deserialize } from './utils/msgpack';
+export { ICache, MemoryCache, RedisCache, PostgresCache, MongoCache, QueueCacheFactory } from './utils/cache';

package/dist/index.js

@@ -0,0 +1,37 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QueueCacheFactory = exports.MongoCache = exports.PostgresCache = exports.RedisCache = exports.MemoryCache = exports.deserialize = exports.serialize = exports.Worker = exports.MongoQueueAdapter = exports.RedisQueueAdapter = exports.PostgresQueueAdapter = exports.QueueFactory = void 0;
+__exportStar(require("./types"), exports);
+var factory_1 = require("./factory");
+Object.defineProperty(exports, "QueueFactory", { enumerable: true, get: function () { return factory_1.QueueFactory; } });
+var postgres_1 = require("./adapters/postgres");
+Object.defineProperty(exports, "PostgresQueueAdapter", { enumerable: true, get: function () { return postgres_1.PostgresQueueAdapter; } });
+var redis_1 = require("./adapters/redis");
+Object.defineProperty(exports, "RedisQueueAdapter", { enumerable: true, get: function () { return redis_1.RedisQueueAdapter; } });
+var mongodb_1 = require("./adapters/mongodb");
+Object.defineProperty(exports, "MongoQueueAdapter", { enumerable: true, get: function () { return mongodb_1.MongoQueueAdapter; } });
+var worker_1 = require("./worker");
+Object.defineProperty(exports, "Worker", { enumerable: true, get: function () { return worker_1.Worker; } });
+var msgpack_1 = require("./utils/msgpack");
+Object.defineProperty(exports, "serialize", { enumerable: true, get: function () { return msgpack_1.serialize; } });
+Object.defineProperty(exports, "deserialize", { enumerable: true, get: function () { return msgpack_1.deserialize; } });
+var cache_1 = require("./utils/cache");
+Object.defineProperty(exports, "MemoryCache", { enumerable: true, get: function () { return cache_1.MemoryCache; } });
+Object.defineProperty(exports, "RedisCache", { enumerable: true, get: function () { return cache_1.RedisCache; } });
+Object.defineProperty(exports, "PostgresCache", { enumerable: true, get: function () { return cache_1.PostgresCache; } });
+Object.defineProperty(exports, "MongoCache", { enumerable: true, get: function () { return cache_1.MongoCache; } });
+Object.defineProperty(exports, "QueueCacheFactory", { enumerable: true, get: function () { return cache_1.QueueCacheFactory; } });

package/dist/types.d.ts

@@ -0,0 +1,37 @@
+export interface Job<T = any> {
+    id: string;
+    queue: string;
+    payload: T;
+    status: 'pending' | 'processing' | 'completed' | 'failed';
+    priority: number;
+    attempts: number;
+    maxAttempts: number;
+    runAt: Date;
+    lockedAt?: Date;
+    completedAt?: Date;
+    failedAt?: Date;
+    error?: string;
+    workerId?: string;
+    expiresAt?: Date;
+}
+export interface EnqueueOptions {
+    id?: string;
+    priority?: number;
+    maxAttempts?: number;
+    runAt?: Date;
+    delayMs?: number;
+}
+export interface QueueAdapterOptions {
+    completedJobTtlMs?: number;
+    failedJobTtlMs?: number;
+    lockTimeoutMs?: number;
+    pruneIntervalMs?: number;
+}
+export interface IQueueAdapter {
+    init?(): Promise<void>;
+    enqueue<T>(queueName: string, payload: T, options?: EnqueueOptions): Promise<Job<T>>;
+    claimJob(queueName: string, workerId: string): Promise<Job | null>;
+    completeJob(jobId: string): Promise<void>;
+    failJob(jobId: string, error: string): Promise<void>;
+    close(): Promise<void>;
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils/cache.d.ts

@@ -0,0 +1,40 @@
+import Redis from 'ioredis';
+import { Pool } from 'pg';
+import { MongoClient, Collection } from 'mongodb';
+export interface ICache {
+    get<T>(key: string): Promise<T | null>;
+    set<T>(key: string, value: T, ttlMs?: number): Promise<void>;
+    del(key: string): Promise<void>;
+}
+export declare class MemoryCache implements ICache {
+    private store;
+    get<T>(key: string): Promise<T | null>;
+    set<T>(key: string, value: T, ttlMs?: number): Promise<void>;
+    del(key: string): Promise<void>;
+}
+export declare class RedisCache implements ICache {
+    private redis;
+    constructor(redisOrUrl: Redis | string);
+    get<T>(key: string): Promise<T | null>;
+    set<T>(key: string, value: T, ttlMs?: number): Promise<void>;
+    del(key: string): Promise<void>;
+}
+export declare class PostgresCache implements ICache {
+    private pool;
+    constructor(poolOrConnectionString: Pool | string);
+    init(): Promise<void>;
+    get<T>(key: string): Promise<T | null>;
+    set<T>(key: string, value: T, ttlMs?: number): Promise<void>;
+    del(key: string): Promise<void>;
+}
+export declare class MongoCache implements ICache {
+    private collection;
+    constructor(clientOrCollection: MongoClient | Collection<any>, dbName?: string);
+    init(): Promise<void>;
+    get<T>(key: string): Promise<T | null>;
+    set<T>(key: string, value: T, ttlMs?: number): Promise<void>;
+    del(key: string): Promise<void>;
+}
+export declare class QueueCacheFactory {
+    static create(connectionString: string): Promise<ICache>;
+}

package/dist/utils/cache.js

@@ -0,0 +1,172 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.QueueCacheFactory = exports.MongoCache = exports.PostgresCache = exports.RedisCache = exports.MemoryCache = void 0;
+const ioredis_1 = __importDefault(require("ioredis"));
+const pg_1 = require("pg");
+const mongodb_1 = require("mongodb");
+class MemoryCache {
+    store = new Map();
+    async get(key) {
+        const item = this.store.get(key);
+        if (!item)
+            return null;
+        if (Date.now() > item.expiresAt) {
+            this.store.delete(key);
+            return null;
+        }
+        return item.value;
+    }
+    async set(key, value, ttlMs = 60000) {
+        this.store.set(key, { value, expiresAt: Date.now() + ttlMs });
+    }
+    async del(key) {
+        this.store.delete(key);
+    }
+}
+exports.MemoryCache = MemoryCache;
+class RedisCache {
+    redis;
+    constructor(redisOrUrl) {
+        if (typeof redisOrUrl === 'string') {
+            this.redis = new ioredis_1.default(redisOrUrl);
+        }
+        else {
+            this.redis = redisOrUrl;
+        }
+    }
+    async get(key) {
+        const data = await this.redis.get(key);
+        if (!data)
+            return null;
+        try {
+            return JSON.parse(data);
+        }
+        catch {
+            return data;
+        }
+    }
+    async set(key, value, ttlMs = 60000) {
+        const data = typeof value === 'string' ? value : JSON.stringify(value);
+        await this.redis.set(key, data, 'PX', ttlMs);
+    }
+    async del(key) {
+        await this.redis.del(key);
+    }
+}
+exports.RedisCache = RedisCache;
+class PostgresCache {
+    pool;
+    constructor(poolOrConnectionString) {
+        if (typeof poolOrConnectionString === 'string') {
+            this.pool = new pg_1.Pool({ connectionString: poolOrConnectionString });
+        }
+        else {
+            this.pool = poolOrConnectionString;
+        }
+    }
+    async init() {
+        const query = `
+      CREATE TABLE IF NOT EXISTS queue_cache (
+        key VARCHAR(255) PRIMARY KEY,
+        value TEXT NOT NULL,
+        expires_at TIMESTAMP WITH TIME ZONE NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_cache_expires ON queue_cache (expires_at);
+    `;
+        await this.pool.query(query);
+    }
+    async get(key) {
+        const query = `
+      SELECT value, expires_at FROM queue_cache
+      WHERE key = $1 AND expires_at > NOW();
+    `;
+        const res = await this.pool.query(query, [key]);
+        if (res.rows.length === 0) {
+            return null;
+        }
+        try {
+            return JSON.parse(res.rows[0].value);
+        }
+        catch {
+            return res.rows[0].value;
+        }
+    }
+    async set(key, value, ttlMs = 60000) {
+        const expiresAt = new Date(Date.now() + ttlMs);
+        const serializedValue = typeof value === 'string' ? value : JSON.stringify(value);
+        const query = `
+      INSERT INTO queue_cache (key, value, expires_at)
+      VALUES ($1, $2, $3)
+      ON CONFLICT (key) DO UPDATE
+      SET value = EXCLUDED.value, expires_at = EXCLUDED.expires_at;
+    `;
+        await this.pool.query(query, [key, serializedValue, expiresAt]);
+    }
+    async del(key) {
+        const query = `DELETE FROM queue_cache WHERE key = $1;`;
+        await this.pool.query(query, [key]);
+    }
+}
+exports.PostgresCache = PostgresCache;
+class MongoCache {
+    collection;
+    constructor(clientOrCollection, dbName = 'queue_tool') {
+        if (clientOrCollection instanceof mongodb_1.MongoClient) {
+            this.collection = clientOrCollection.db(dbName).collection('queue_cache');
+        }
+        else {
+            this.collection = clientOrCollection;
+        }
+    }
+    async init() {
+        await this.collection.createIndex({ expiresAt: 1 }, { name: 'idx_cache_ttl', expireAfterSeconds: 0 });
+    }
+    async get(key) {
+        const doc = await this.collection.findOne({
+            _id: key,
+            expiresAt: { $gt: new Date() }
+        });
+        if (!doc)
+            return null;
+        return doc.value;
+    }
+    async set(key, value, ttlMs = 60000) {
+        const expiresAt = new Date(Date.now() + ttlMs);
+        await this.collection.updateOne({ _id: key }, {
+            $set: {
+                value: value,
+                expiresAt: expiresAt
+            }
+        }, { upsert: true });
+    }
+    async del(key) {
+        await this.collection.deleteOne({ _id: key });
+    }
+}
+exports.MongoCache = MongoCache;
+class QueueCacheFactory {
+    static async create(connectionString) {
+        if (connectionString.startsWith('postgres://') || connectionString.startsWith('postgresql://')) {
+            const cache = new PostgresCache(connectionString);
+            await cache.init();
+            return cache;
+        }
+        else if (connectionString.startsWith('redis://') || connectionString.startsWith('rediss://')) {
+            return new RedisCache(connectionString);
+        }
+        else if (connectionString.startsWith('mongodb://') || connectionString.startsWith('mongodb+srv://')) {
+            const client = new mongodb_1.MongoClient(connectionString);
+            await client.connect();
+            const cache = new MongoCache(client);
+            await cache.init();
+            return cache;
+        }
+        else {
+            return new MemoryCache();
+        }
+    }
+}
+exports.QueueCacheFactory = QueueCacheFactory;

package/dist/utils/cache.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/utils/cache.test.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const cache_1 = require("./cache");
+describe('MemoryCache', () => {
+    let cache;
+    beforeEach(() => {
+        cache = new cache_1.MemoryCache();
+    });
+    it('should set and get values correctly', async () => {
+        await cache.set('mykey', 'myvalue');
+        const val = await cache.get('mykey');
+        expect(val).toBe('myvalue');
+    });
+    it('should return null for non-existent keys', async () => {
+        const val = await cache.get('non-existent');
+        expect(val).toBeNull();
+    });
+    it('should delete keys successfully', async () => {
+        await cache.set('mykey', 'myvalue');
+        await cache.del('mykey');
+        const val = await cache.get('mykey');
+        expect(val).toBeNull();
+    });
+    it('should respect TTL expiration', async () => {
+        jest.useFakeTimers();
+        await cache.set('tempkey', 'tempvalue', 100); // 100ms TTL
+        // Immediately gets value
+        expect(await cache.get('tempkey')).toBe('tempvalue');
+        // Advance time by 101ms
+        jest.advanceTimersByTime(101);
+        // Should be expired now
+        expect(await cache.get('tempkey')).toBeNull();
+        jest.useRealTimers();
+    });
+});

package/dist/utils/logger.d.ts

@@ -0,0 +1,15 @@
+export interface LogContext {
+    queue?: string;
+    jobId?: string;
+    workerId?: string;
+    error?: string;
+    [key: string]: any;
+}
+declare class JsonLogger {
+    private formatLog;
+    info(message: string, context?: LogContext): void;
+    error(message: string, context?: LogContext): void;
+    warn(message: string, context?: LogContext): void;
+}
+export declare const logger: JsonLogger;
+export {};

package/dist/utils/logger.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.logger = void 0;
+class JsonLogger {
+    formatLog(level, message, context) {
+        return JSON.stringify({
+            timestamp: new Date().toISOString(),
+            level,
+            message,
+            ...context
+        });
+    }
+    info(message, context) {
+        console.log(this.formatLog('INFO', message, context));
+    }
+    error(message, context) {
+        console.error(this.formatLog('ERROR', message, context));
+    }
+    warn(message, context) {
+        console.warn(this.formatLog('WARN', message, context));
+    }
+}
+exports.logger = new JsonLogger();

package/dist/utils/msgpack.d.ts

@@ -0,0 +1,2 @@
+export declare function serialize(data: any): Buffer;
+export declare function deserialize<T = any>(buffer: Buffer | Uint8Array): T;

package/dist/utils/msgpack.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.serialize = serialize;
+exports.deserialize = deserialize;
+const msgpack_1 = require("@msgpack/msgpack");
+function serialize(data) {
+    const encoded = (0, msgpack_1.encode)(data);
+    return Buffer.from(encoded.buffer, encoded.byteOffset, encoded.byteLength);
+}
+function deserialize(buffer) {
+    const uint8 = buffer instanceof Buffer ? new Uint8Array(buffer) : buffer;
+    return (0, msgpack_1.decode)(uint8);
+}

package/dist/utils/msgpack.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/utils/msgpack.test.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const msgpack_1 = require("./msgpack");
+describe('MessagePack Utility', () => {
+    it('should serialize and deserialize a simple object', () => {
+        const data = { foo: 'bar', age: 42 };
+        const serialized = (0, msgpack_1.serialize)(data);
+        expect(serialized).toBeInstanceOf(Buffer);
+        const deserialized = (0, msgpack_1.deserialize)(serialized);
+        expect(deserialized).toEqual(data);
+    });
+    it('should handle complex nested objects and arrays', () => {
+        const data = {
+            id: 'img-10293',
+            operations: ['resize', 'compress'],
+            metadata: {
+                width: 1920,
+                height: 1080,
+                tags: ['png', 'premium']
+            },
+            active: true
+        };
+        const serialized = (0, msgpack_1.serialize)(data);
+        const deserialized = (0, msgpack_1.deserialize)(serialized);
+        expect(deserialized).toEqual(data);
+    });
+    it('should handle primitives and null values', () => {
+        expect((0, msgpack_1.deserialize)((0, msgpack_1.serialize)(null))).toBeNull();
+        expect((0, msgpack_1.deserialize)((0, msgpack_1.serialize)(123.45))).toBe(123.45);
+        expect((0, msgpack_1.deserialize)((0, msgpack_1.serialize)('hello world'))).toBe('hello world');
+        expect((0, msgpack_1.deserialize)((0, msgpack_1.serialize)(false))).toBe(false);
+    });
+});

package/dist/worker.d.ts

@@ -0,0 +1,23 @@
+import { IQueueAdapter, Job } from './types';
+export interface WorkerOptions {
+    concurrency?: number;
+    pollIntervalMs?: number;
+    taskNameKey?: string;
+}
+export declare class Worker {
+    private queueName;
+    private connectionOrAdapter;
+    private processor;
+    private adapter;
+    private running;
+    private workerId;
+    private concurrency;
+    private pollIntervalMs;
+    private taskNameKey;
+    private ownAdapter;
+    private loops;
+    constructor(queueName: string, connectionOrAdapter: string | IQueueAdapter, processor: ((job: Job) => Promise<void>) | Record<string, (job: Job) => Promise<void>>, options?: WorkerOptions);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    private runLoop;
+}