redis-smq-common 1.0.1 → 1.0.4

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # CHANGELOG
 
+## 1.0.4 (2022-08-24)
+
+* test(redis-client): cover srem command (caf3837)
+* feat(redis-client): add support for srem command (7ddaf24)
+
+
+## 1.0.3 (2022-08-16)
+
+* [RedisClient] Update tests (6f1fd51)
+* [RedisClient] Make validateRedisVersion() public (c2fccb3)
+* [RedisClient] Support MATCH and COUNT options for sscan (12b24ac)
+* [RedisClient] Fallback to smembers when sscan is not supported (8a4409f)
+* [RedisClient] Test sscan command (17b8279)
+* [RedisClient] Add sscan command (b771873)
+
+## 1.0.2 (2022-08-10)
+
+* Update LockManager tests (321c8c4)
+* Fix "LockManagerExtendError: Acquired lock could not be extended" (fa3a8e5)
+
 ## 1.0.1 (2022-07-07)
 
 * Remove unused WorkerRunnerError (48e7206)

package/dist/src/lock-manager/errors/lock-manager-method-not-allowed.error.d.ts

@@ -0,0 +1,4 @@
+import { LockManagerError } from './lock-manager.error';
+export declare class LockManagerMethodNotAllowedError extends LockManagerError {
+    constructor(message?: string);
+}

package/dist/src/lock-manager/errors/lock-manager-method-not-allowed.error.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LockManagerMethodNotAllowedError = void 0;
+const lock_manager_error_1 = require("./lock-manager.error");
+class LockManagerMethodNotAllowedError extends lock_manager_error_1.LockManagerError {
+    constructor(message = `This method can not be used when autoExtend is enabled`) {
+        super(message);
+    }
+}
+exports.LockManagerMethodNotAllowedError = LockManagerMethodNotAllowedError;
+//# sourceMappingURL=lock-manager-method-not-allowed.error.js.map

package/dist/src/lock-manager/errors/lock-manager-not-acquired.error.d.ts

@@ -0,0 +1,4 @@
+import { LockManagerError } from './lock-manager.error';
+export declare class LockManagerNotAcquiredError extends LockManagerError {
+    constructor(message?: string);
+}

package/dist/src/lock-manager/errors/lock-manager-not-acquired.error.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LockManagerNotAcquiredError = void 0;
+const lock_manager_error_1 = require("./lock-manager.error");
+class LockManagerNotAcquiredError extends lock_manager_error_1.LockManagerError {
+    constructor(message = `Can not extend a lock which has not been yet acquired. Maybe a pending operation is in progress.`) {
+        super(message);
+    }
+}
+exports.LockManagerNotAcquiredError = LockManagerNotAcquiredError;
+//# sourceMappingURL=lock-manager-not-acquired.error.js.map

package/dist/src/lock-manager/errors/lock-manager-not-released.error.d.ts

@@ -0,0 +1,4 @@
+import { LockManagerError } from './lock-manager.error';
+export declare class LockManagerNotReleasedError extends LockManagerError {
+    constructor(message?: string);
+}

package/dist/src/lock-manager/errors/lock-manager-not-released.error.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LockManagerNotReleasedError = void 0;
+const lock_manager_error_1 = require("./lock-manager.error");
+class LockManagerNotReleasedError extends lock_manager_error_1.LockManagerError {
+    constructor(message = `A lock has been already obtained but not yet released or maybe a pending operation is in progress.`) {
+        super(message);
+    }
+}
+exports.LockManagerNotReleasedError = LockManagerNotReleasedError;
+//# sourceMappingURL=lock-manager-not-released.error.js.map

package/dist/src/lock-manager/lock-manager.d.ts

@@ -1,12 +1,13 @@
 /// <reference types="node" />
 import { ICallback } from '../../types';
 import { RedisClient } from '../redis-client/redis-client';
-declare enum ELockStatus {
+export declare enum ELockStatus {
     unlocked = 0,
     locking = 1,
     locked = 2,
     releasing = 3,
-    extending = 4
+    extending = 4,
+    extended = 5
 }
 export declare enum ELuaScript {
     RELEASE_LOCK = "RELEASE_LOCK",
@@ -22,16 +23,19 @@ export declare class LockManager {
     protected status: ELockStatus;
     protected lockingTimer: NodeJS.Timeout | null;
     protected autoExtendTimer: NodeJS.Timeout | null;
-    constructor(redisClient: RedisClient, lockKey: string, ttl: number, retryOnFail?: boolean, autoExtend?: boolean);
+    protected throwExceptions: boolean;
+    constructor(redisClient: RedisClient, lockKey: string, ttl: number, retryOnFail?: boolean, autoExtend?: boolean, throwExceptions?: boolean);
     protected resetTimers(): void;
     protected setUnlocked(): void;
     protected setLocked(): void;
+    protected setExtended(): void;
     protected extend(cb: ICallback<void>): void;
     protected runAutoExtendTimer(): void;
     acquireLock(cb: ICallback<void>): void;
     extendLock(cb: ICallback<void>): void;
     releaseLock(cb: ICallback<void>): void;
+    acquireOrExtend(cb: ICallback<ELockStatus>): void;
     isLocked(): boolean;
+    isReleased(): boolean;
     getId(): string;
 }
-export {};

package/dist/src/lock-manager/lock-manager.js

@@ -1,13 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LockManager = exports.ELuaScript = void 0;
+exports.LockManager = exports.ELuaScript = exports.ELockStatus = void 0;
 const redis_client_1 = require("../redis-client/redis-client");
 const uuid_1 = require("uuid");
-const lock_manager_error_1 = require("./errors/lock-manager.error");
 const lock_manager_abort_error_1 = require("./errors/lock-manager-abort.error");
 const lock_manager_extend_error_1 = require("./errors/lock-manager-extend.error");
 const lock_manager_acquire_error_1 = require("./errors/lock-manager-acquire.error");
 const fs = require("fs");
+const lock_manager_method_not_allowed_error_1 = require("./errors/lock-manager-method-not-allowed.error");
+const lock_manager_not_acquired_error_1 = require("./errors/lock-manager-not-acquired.error");
+const lock_manager_not_released_error_1 = require("./errors/lock-manager-not-released.error");
 var ELockStatus;
 (function (ELockStatus) {
     ELockStatus[ELockStatus["unlocked"] = 0] = "unlocked";
@@ -15,7 +17,8 @@ var ELockStatus;
     ELockStatus[ELockStatus["locked"] = 2] = "locked";
     ELockStatus[ELockStatus["releasing"] = 3] = "releasing";
     ELockStatus[ELockStatus["extending"] = 4] = "extending";
-})(ELockStatus || (ELockStatus = {}));
+    ELockStatus[ELockStatus["extended"] = 5] = "extended";
+})(ELockStatus = exports.ELockStatus || (exports.ELockStatus = {}));
 var ELuaScript;
 (function (ELuaScript) {
     ELuaScript["RELEASE_LOCK"] = "RELEASE_LOCK";
@@ -24,16 +27,18 @@ var ELuaScript;
 redis_client_1.RedisClient.addScript(ELuaScript.RELEASE_LOCK, fs.readFileSync(`${__dirname}/redis-client/lua/release-lock.lua`).toString());
 redis_client_1.RedisClient.addScript(ELuaScript.EXTEND_LOCK, fs.readFileSync(`${__dirname}/redis-client/lua/extend-lock.lua`).toString());
 class LockManager {
-    constructor(redisClient, lockKey, ttl, retryOnFail = false, autoExtend = false) {
+    constructor(redisClient, lockKey, ttl, retryOnFail = false, autoExtend = false, throwExceptions = true) {
         this.status = ELockStatus.unlocked;
         this.lockingTimer = null;
         this.autoExtendTimer = null;
+        this.throwExceptions = true;
         this.lockKey = lockKey;
         this.ttl = ttl;
         this.retryOnFail = retryOnFail;
         this.lockId = (0, uuid_1.v4)();
         this.redisClient = redisClient;
         this.autoExtend = autoExtend;
+        this.throwExceptions = throwExceptions;
     }
     resetTimers() {
         if (this.lockingTimer) {
@@ -51,10 +56,12 @@ class LockManager {
     setLocked() {
         this.status = ELockStatus.locked;
     }
+    setExtended() {
+        this.status = ELockStatus.extended;
+    }
     extend(cb) {
-        if (this.status !== ELockStatus.locked) {
-            cb(new lock_manager_error_1.LockManagerError('The lock is currently not acquired or a pending operation is in progress'));
-        }
+        if (!this.isLocked())
+            cb(new lock_manager_not_acquired_error_1.LockManagerNotAcquiredError());
         else {
             this.status = ELockStatus.extending;
             this.redisClient.runScript(ELuaScript.EXTEND_LOCK, [this.lockKey], [this.lockId, this.ttl], (err, reply) => {
@@ -67,7 +74,7 @@ class LockManager {
                             cb(new lock_manager_extend_error_1.LockManagerExtendError());
                         }
                         else {
-                            this.setLocked();
+                            this.setExtended();
                             cb();
                         }
                     }
@@ -83,14 +90,14 @@ class LockManager {
         this.autoExtendTimer = setTimeout(() => this.extend((err) => {
             if (!err)
                 this.runAutoExtendTimer();
-            else if (!(err instanceof lock_manager_abort_error_1.LockManagerAbortError))
+            else if (this.throwExceptions &&
+                !(err instanceof lock_manager_abort_error_1.LockManagerAbortError))
                 throw err;
         }), ms);
     }
     acquireLock(cb) {
-        if (this.status !== ELockStatus.unlocked) {
-            cb(new lock_manager_error_1.LockManagerError(`The lock is currently not released or a pending operation is in progress`));
-        }
+        if (!this.isReleased())
+            cb(new lock_manager_not_released_error_1.LockManagerNotReleasedError());
         else {
             this.status = ELockStatus.locking;
             const lock = () => {
@@ -134,18 +141,17 @@ class LockManager {
         }
     }
     extendLock(cb) {
-        if (this.autoExtend) {
-            cb(new lock_manager_error_1.LockManagerError(`Can not extend a lock when autoExtend is enabled`));
-        }
+        if (this.autoExtend)
+            cb(new lock_manager_method_not_allowed_error_1.LockManagerMethodNotAllowedError());
         else
             this.extend(cb);
     }
     releaseLock(cb) {
         const status = this.status;
-        if (status === ELockStatus.releasing)
-            cb(new lock_manager_error_1.LockManagerError('A pending releaseLock() call is in progress'));
-        else if (status === ELockStatus.unlocked)
+        if (status === ELockStatus.unlocked)
             cb();
+        else if (!this.isLocked())
+            cb(new lock_manager_not_acquired_error_1.LockManagerNotAcquiredError());
         else {
             this.resetTimers();
             this.status = ELockStatus.releasing;
@@ -159,9 +165,38 @@ class LockManager {
             });
         }
     }
+    acquireOrExtend(cb) {
+        if (this.autoExtend)
+            cb(new lock_manager_method_not_allowed_error_1.LockManagerMethodNotAllowedError());
+        else {
+            const lock = () => {
+                this.acquireLock((err) => {
+                    if (err)
+                        cb(err);
+                    else
+                        cb(null, ELockStatus.locked);
+                });
+            };
+            if (this.isLocked())
+                this.extend((err) => {
+                    if (err) {
+                        if (err instanceof lock_manager_extend_error_1.LockManagerExtendError)
+                            lock();
+                        else
+                            cb(err);
+                    }
+                    else
+                        cb(null, ELockStatus.extended);
+                });
+            else
+                lock();
+        }
+    }
     isLocked() {
-        return (this.status === ELockStatus.locked ||
-            this.status === ELockStatus.extending);
+        return (this.status === ELockStatus.locked || this.status === ELockStatus.extended);
+    }
+    isReleased() {
+        return this.status === ELockStatus.unlocked;
     }
     getId() {
         return this.lockId;

package/dist/src/redis-client/clients/ioredis-client.d.ts

@@ -17,6 +17,10 @@ export declare class IoredisClient extends RedisClient {
     watch(args: string[], cb: ICallback<string>): void;
     unwatch(cb: ICallback<string>): void;
     sismember(key: string, member: string, cb: ICallback<number>): void;
+    sscan(key: string, options: {
+        MATCH?: string;
+        COUNT?: number;
+    }, cb: ICallback<string[]>): void;
     zcard(key: string, cb: ICallback<number>): void;
     zrange(key: string, min: number, max: number, cb: ICallback<string[]>): void;
     psubscribe(pattern: string): void;
@@ -26,6 +30,7 @@ export declare class IoredisClient extends RedisClient {
     zrangebyscore(key: string, min: number | string, max: number | string, cb: ICallback<string[]>): void;
     smembers(key: string, cb: ICallback<string[]>): void;
     sadd(key: string, member: string, cb: ICallback<number>): void;
+    srem(key: string, member: string, cb: ICallback<number>): void;
     hgetall(key: string, cb: ICallback<Record<string, string>>): void;
     hget(key: string, field: string, cb: ICallback<string | null>): void;
     hset(key: string, field: string, value: string, cb: ICallback<number>): void;

package/dist/src/redis-client/clients/ioredis-client.js

@@ -47,6 +47,28 @@ class IoredisClient extends redis_client_1.RedisClient {
     sismember(key, member, cb) {
         this.client.sismember(key, member, cb);
     }
+    sscan(key, options, cb) {
+        const result = new Set();
+        const iterate = (position) => {
+            const args = [key, position];
+            if (options.MATCH)
+                args.push('MATCH', options.MATCH);
+            if (options.COUNT)
+                args.push('COUNT', String(options.COUNT));
+            this.client.sscan(...args, (err, [cursor, items]) => {
+                if (err)
+                    cb(err);
+                else {
+                    items.forEach((i) => result.add(i));
+                    if (cursor === '0')
+                        cb(null, [...result]);
+                    else
+                        iterate(cursor);
+                }
+            });
+        };
+        iterate('0');
+    }
     zcard(key, cb) {
         this.client.zcard(key, cb);
     }
@@ -74,6 +96,9 @@ class IoredisClient extends redis_client_1.RedisClient {
     sadd(key, member, cb) {
         this.client.sadd(key, member, cb);
     }
+    srem(key, member, cb) {
+        this.client.srem(key, member, cb);
+    }
     hgetall(key, cb) {
         this.client.hgetall(key, cb);
     }

package/dist/src/redis-client/clients/node-redis-v3-client.d.ts

@@ -17,6 +17,10 @@ export declare class NodeRedisV3Client extends RedisClient {
     watch(args: string[], cb: ICallback<string>): void;
     unwatch(cb: ICallback<string>): void;
     sismember(key: string, member: string, cb: ICallback<number>): void;
+    sscan(key: string, options: {
+        MATCH?: string;
+        COUNT?: number;
+    }, cb: ICallback<string[]>): void;
     zcard(key: string, cb: ICallback<number>): void;
     zrange(key: string, min: number, max: number, cb: ICallback<string[]>): void;
     psubscribe(pattern: string): void;
@@ -26,6 +30,7 @@ export declare class NodeRedisV3Client extends RedisClient {
     zrangebyscore(key: string, min: number | string, max: number | string, cb: ICallback<string[]>): void;
     smembers(key: string, cb: ICallback<string[]>): void;
     sadd(key: string, member: string, cb: ICallback<number>): void;
+    srem(key: string, member: string, cb: ICallback<number>): void;
     hgetall(key: string, cb: ICallback<Record<string, string>>): void;
     hget(key: string, field: string, cb: ICallback<string | null>): void;
     hset(key: string, field: string, value: string, cb: ICallback<number>): void;

package/dist/src/redis-client/clients/node-redis-v3-client.js

@@ -74,6 +74,28 @@ class NodeRedisV3Client extends redis_client_1.RedisClient {
     sismember(key, member, cb) {
         this.client.sismember(key, member, cb);
     }
+    sscan(key, options, cb) {
+        const result = new Set();
+        const iterate = (position) => {
+            const args = [key, position];
+            if (options.MATCH)
+                args.push('MATCH', options.MATCH);
+            if (options.COUNT)
+                args.push('COUNT', String(options.COUNT));
+            this.client.sscan(...args, (err, [cursor, items]) => {
+                if (err)
+                    cb(err);
+                else {
+                    items.forEach((i) => result.add(i));
+                    if (cursor === '0')
+                        cb(null, [...result]);
+                    else
+                        iterate(cursor);
+                }
+            });
+        };
+        iterate('0');
+    }
     zcard(key, cb) {
         this.client.zcard(key, cb);
     }
@@ -101,6 +123,9 @@ class NodeRedisV3Client extends redis_client_1.RedisClient {
     sadd(key, member, cb) {
         this.client.sadd(key, member, cb);
     }
+    srem(key, member, cb) {
+        this.client.srem(key, member, cb);
+    }
     hgetall(key, cb) {
         this.client.hgetall(key, (err, reply) => {
             if (err)

package/dist/src/redis-client/clients/node-redis-v4-client.d.ts

@@ -25,7 +25,12 @@ export declare class NodeRedisV4Client extends RedisClient {
     unsubscribe(channel: string): void;
     zrangebyscore(key: string, min: number | string, max: number | string, cb: ICallback<string[]>): void;
     smembers(key: string, cb: ICallback<string[]>): void;
+    sscan(key: string, options: {
+        MATCH?: string;
+        COUNT?: number;
+    }, cb: ICallback<string[]>): void;
     sadd(key: string, member: string, cb: ICallback<number>): void;
+    srem(key: string, member: string, cb: ICallback<number>): void;
     hgetall(key: string, cb: ICallback<Record<string, string>>): void;
     hget(key: string, field: string, cb: ICallback<string | null>): void;
     hset(key: string, field: string, value: string, cb: ICallback<number>): void;

package/dist/src/redis-client/clients/node-redis-v4-client.js

@@ -94,12 +94,39 @@ class NodeRedisV4Client extends redis_client_1.RedisClient {
             .then((reply) => cb(null, reply))
             .catch(cb);
     }
+    sscan(key, options, cb) {
+        const result = new Set();
+        const iterate = (position) => {
+            const args = [
+                key,
+                position,
+                options,
+            ];
+            this.client
+                .sScan(...args)
+                .then(({ cursor, members }) => {
+                members.forEach((i) => result.add(i));
+                if (cursor === 0)
+                    cb(null, [...result]);
+                else
+                    iterate(cursor);
+            })
+                .catch(cb);
+        };
+        iterate(0);
+    }
     sadd(key, member, cb) {
         this.client
             .sAdd(key, member)
             .then((reply) => cb(null, reply))
             .catch(cb);
     }
+    srem(key, member, cb) {
+        this.client
+            .sRem(key, member)
+            .then((reply) => cb(null, reply))
+            .catch(cb);
+    }
     hgetall(key, cb) {
         this.client
             .hGetAll(key)

package/dist/src/redis-client/redis-client.d.ts

@@ -12,7 +12,7 @@ export declare abstract class RedisClient extends EventEmitter {
     protected static scriptsLoaded: boolean;
     protected static luaScripts: LuaScripts;
     protected connectionClosed: boolean;
-    protected validateRedisVersion(major: number, feature?: number, minor?: number): boolean;
+    validateRedisVersion(major: number, feature?: number, minor?: number): boolean;
     abstract set(key: string, value: string, options: {
         expire?: {
             mode: 'EX' | 'PX';
@@ -33,7 +33,13 @@ export declare abstract class RedisClient extends EventEmitter {
     abstract unsubscribe(channel: string): void;
     abstract zrangebyscore(key: string, min: number | string, max: number | string, cb: ICallback<string[]>): void;
     abstract smembers(key: string, cb: ICallback<string[]>): void;
+    abstract sscan(key: string, options: {
+        MATCH?: string;
+        COUNT?: number;
+    }, cb: ICallback<string[]>): void;
+    sscanFallback(key: string, cb: ICallback<string[]>): void;
     abstract sadd(key: string, member: string, cb: ICallback<number>): void;
+    abstract srem(key: string, member: string, cb: ICallback<number>): void;
     abstract hgetall(key: string, cb: ICallback<Record<string, string>>): void;
     abstract hget(key: string, field: string, cb: ICallback<string | null>): void;
     abstract hset(key: string, field: string, value: string, cb: ICallback<number>): void;

package/dist/src/redis-client/redis-client.js

@@ -25,6 +25,12 @@ class RedisClient extends events_1.EventEmitter {
                 RedisClient.redisServerVersion[1] >= feature &&
                 RedisClient.redisServerVersion[2] >= minor));
     }
+    sscanFallback(key, cb) {
+        if (this.validateRedisVersion(2, 8))
+            this.sscan(key, {}, cb);
+        else
+            this.smembers(key, cb);
+    }
     zpophgetrpush(source, sourceHash, destination, cb) {
         this.runScript(ELuaScriptName.ZPOPHGETRPUSH, [source, sourceHash, destination], [], (err, res) => {
             if (err)

package/dist/src/worker/worker-runner/worker-runner.js

@@ -14,16 +14,12 @@ class WorkerRunner extends events_1.EventEmitter {
         this.onTick = () => {
             async_1.async.waterfall([
                 (cb) => {
-                    if (!this.lockManager.isLocked()) {
-                        this.lockManager.acquireLock((err) => {
-                            if (!err) {
-                                this.logger.info(`Workers are exclusively running from this instance (Lock ID ${this.lockManager.getId()}).`);
-                            }
-                            cb(err);
-                        });
-                    }
-                    else
-                        cb();
+                    this.lockManager.acquireOrExtend((err, status) => {
+                        if (status === lock_manager_1.ELockStatus.locked) {
+                            this.logger.info(`Workers are exclusively running from this instance (Lock ID ${this.lockManager.getId()}).`);
+                        }
+                        cb(err);
+                    });
                 },
                 (cb) => {
                     this.workerPool.work(cb);
@@ -58,7 +54,7 @@ class WorkerRunner extends events_1.EventEmitter {
         this.powerManager = new power_manager_1.PowerManager();
         this.redisClient = redisClient;
         this.logger = logger;
-        this.lockManager = new lock_manager_1.LockManager(redisClient, keyLock, 10000, false, true);
+        this.lockManager = new lock_manager_1.LockManager(redisClient, keyLock, 60000);
         this.ticker = new ticker_1.Ticker(this.onTick);
         this.workerPool = workerPool;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "redis-smq-common",
-  "version": "1.0.1",
+  "version": "1.0.4",
   "description": "RedisSMQ shared components that may be used by integrated applications and extensions.",
   "author": "Weyoss <weyoss@protonmail.com>",
   "license": "MIT",