queue-tool 1.0.0

package/dist/worker.js

@@ -0,0 +1,100 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Worker = void 0;
+const factory_1 = require("./factory");
+const logger_1 = require("./utils/logger");
+const crypto_1 = require("crypto");
+class Worker {
+    queueName;
+    connectionOrAdapter;
+    processor;
+    adapter;
+    running = false;
+    workerId;
+    concurrency;
+    pollIntervalMs;
+    taskNameKey;
+    ownAdapter = false;
+    loops = [];
+    constructor(queueName, connectionOrAdapter, processor, options) {
+        this.queueName = queueName;
+        this.connectionOrAdapter = connectionOrAdapter;
+        this.processor = processor;
+        this.workerId = `worker-${(0, crypto_1.randomUUID)()}`;
+        this.concurrency = options?.concurrency ?? 1;
+        this.pollIntervalMs = options?.pollIntervalMs ?? 1000;
+        this.taskNameKey = options?.taskNameKey ?? 'task';
+    }
+    async start() {
+        if (this.running) {
+            return;
+        }
+        logger_1.logger.info('Starting worker...', { queue: this.queueName, workerId: this.workerId, concurrency: this.concurrency });
+        if (typeof this.connectionOrAdapter === 'string') {
+            this.adapter = await factory_1.QueueFactory.create(this.connectionOrAdapter);
+            this.ownAdapter = true;
+        }
+        else {
+            this.adapter = this.connectionOrAdapter;
+        }
+        this.running = true;
+        this.loops = [];
+        for (let i = 0; i < this.concurrency; i++) {
+            this.loops.push(this.runLoop(i));
+        }
+    }
+    async stop() {
+        if (!this.running) {
+            return;
+        }
+        logger_1.logger.info('Stopping worker...', { queue: this.queueName, workerId: this.workerId });
+        this.running = false;
+        // Wait for all concurrent loops to finish their current job
+        await Promise.all(this.loops);
+        this.loops = [];
+        if (this.ownAdapter && this.adapter) {
+            await this.adapter.close();
+        }
+        logger_1.logger.info('Worker stopped successfully', { queue: this.queueName, workerId: this.workerId });
+    }
+    async runLoop(loopIndex) {
+        logger_1.logger.info(`Worker loop ${loopIndex} started`, { queue: this.queueName, workerId: this.workerId });
+        while (this.running) {
+            try {
+                const job = await this.adapter.claimJob(this.queueName, this.workerId);
+                if (job) {
+                    logger_1.logger.info(`Worker processing job`, { queue: this.queueName, jobId: job.id, workerId: this.workerId });
+                    try {
+                        if (typeof this.processor === 'function') {
+                            await this.processor(job);
+                        }
+                        else {
+                            const taskName = job.payload && job.payload[this.taskNameKey];
+                            if (taskName && typeof this.processor[taskName] === 'function') {
+                                await this.processor[taskName](job);
+                            }
+                            else {
+                                throw new Error(`No handler registered for task: ${taskName}`);
+                            }
+                        }
+                        await this.adapter.completeJob(job.id);
+                        logger_1.logger.info(`Worker successfully completed job`, { queue: this.queueName, jobId: job.id, workerId: this.workerId });
+                    }
+                    catch (procErr) {
+                        const errMsg = procErr?.message || String(procErr);
+                        logger_1.logger.error(`Worker job execution failed`, { queue: this.queueName, jobId: job.id, workerId: this.workerId, error: errMsg });
+                        await this.adapter.failJob(job.id, errMsg);
+                    }
+                    // Process next job immediately
+                    continue;
+                }
+            }
+            catch (err) {
+                logger_1.logger.error(`Worker loop error`, { queue: this.queueName, workerId: this.workerId, error: err.message });
+            }
+            // No job claimed, wait before polling again
+            await new Promise(resolve => setTimeout(resolve, this.pollIntervalMs));
+        }
+    }
+}
+exports.Worker = Worker;

package/dist/worker.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/worker.test.js

@@ -0,0 +1,89 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const redis_1 = require("./adapters/redis");
+const worker_1 = require("./worker");
+const ioredis_1 = __importDefault(require("ioredis"));
+describe('Worker Integration Tests', () => {
+    let adapter;
+    let redisClient;
+    const queueName = 'test-worker-queue';
+    const connectionString = 'redis://localhost:6379';
+    beforeAll(async () => {
+        redisClient = new ioredis_1.default(connectionString);
+        adapter = new redis_1.RedisQueueAdapter(connectionString);
+        await adapter.init();
+    });
+    afterAll(async () => {
+        await adapter.close();
+        await redisClient.quit();
+    });
+    beforeEach(async () => {
+        const keys = await redisClient.keys(`*${queueName}*`);
+        if (keys.length > 0) {
+            await redisClient.del(...keys);
+        }
+    });
+    test('should process enqueued jobs automatically', async () => {
+        const processedJobs = [];
+        const worker = new worker_1.Worker(queueName, adapter, async (job) => {
+            processedJobs.push(job);
+        }, { pollIntervalMs: 100 });
+        await worker.start();
+        const payload = { test: 'worker-job', val: 99 };
+        await adapter.enqueue(queueName, payload);
+        await new Promise(resolve => setTimeout(resolve, 300));
+        expect(processedJobs.length).toBe(1);
+        expect(processedJobs[0].payload).toEqual(payload);
+        const finalJob = await adapter.getJob(processedJobs[0].id);
+        expect(finalJob.status).toBe('completed');
+        await worker.stop();
+    });
+    test('should route jobs to named task handlers', async () => {
+        const executedTasks = [];
+        const worker = new worker_1.Worker(queueName, adapter, {
+            sendEmail: async (job) => {
+                executedTasks.push(`email:${job.payload.to}`);
+            },
+            generateReport: async (job) => {
+                executedTasks.push(`report:${job.payload.type}`);
+            }
+        }, { pollIntervalMs: 100, taskNameKey: 'action' });
+        await worker.start();
+        // Enqueue jobs with different actions
+        await adapter.enqueue(queueName, { action: 'sendEmail', to: 'test@example.com' });
+        await adapter.enqueue(queueName, { action: 'generateReport', type: 'pdf' });
+        await new Promise(resolve => setTimeout(resolve, 400));
+        expect(executedTasks).toContain('email:test@example.com');
+        expect(executedTasks).toContain('report:pdf');
+        expect(executedTasks.length).toBe(2);
+        await worker.stop();
+    });
+    test('should handle job failure and retries automatically', async () => {
+        let processAttempts = 0;
+        const worker = new worker_1.Worker(queueName, adapter, async (job) => {
+            processAttempts++;
+            throw new Error('processing failed intentionally');
+        }, { pollIntervalMs: 100 });
+        await worker.start();
+        const job = await adapter.enqueue(queueName, { task: 'will-fail' }, { maxAttempts: 2 });
+        await new Promise(resolve => setTimeout(resolve, 200));
+        expect(processAttempts).toBe(1);
+        const checkJob1 = await adapter.getJob(job.id);
+        expect(checkJob1.status).toBe('pending');
+        expect(checkJob1.error).toBe('processing failed intentionally');
+        const jobKey = `job:${job.id}`;
+        const pastTime = Date.now() - 60000;
+        await redisClient.hset(jobKey, 'run_at', pastTime.toString());
+        const score = pastTime - (job.priority * 10000000000);
+        await redisClient.zadd(`queue:${queueName}:pending`, score, job.id);
+        await new Promise(resolve => setTimeout(resolve, 300));
+        expect(processAttempts).toBe(2);
+        const checkJob2 = await adapter.getJob(job.id);
+        expect(checkJob2.status).toBe('failed');
+        expect(checkJob2.error).toBe('processing failed intentionally');
+        await worker.stop();
+    });
+});

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "queue-tool",
+  "version": "1.0.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "jest",
+    "lint": "eslint src --ext .ts"
+  },
+  "dependencies": {
+    "@msgpack/msgpack": "^2.8.0",
+    "ioredis": "^5.4.1",
+    "mongodb": "^6.6.2",
+    "pg": "^8.11.5"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.12",
+    "@types/node": "^20.12.12",
+    "@types/pg": "^8.11.6",
+    "@typescript-eslint/eslint-plugin": "^7.10.0",
+    "@typescript-eslint/parser": "^7.10.0",
+    "eslint": "^8.57.0",
+    "jest": "^29.5.0",
+    "ts-jest": "^29.1.0",
+    "typescript": "^5.4.5"
+  }
+}