redis-abstraction 0.0.2

package/README.md

@@ -0,0 +1,96 @@
+# redis-abstraction
+A Redis client pool with abstraction to different Redis libraries.
+
+This package helps to create redis connections in a connection pool fashion, ideally redis is single threaded so multiple connection or a connection pool for simple commands may not make sense, but there are some conditions in which connection pool makes sense like blocking commands, pub-sub commands etc apart form these cases there are also some fringe cases where if you have a simple command with a lot of data to be serialized on wire/network here multiple connections make sense as redis execution is single threaded but its I/O stack is multi so another command which has relatively small data to be serialized on wire gets stuck behind this big network transfer command ahead of it, It can take advantage of another connection where there is not que before-hand and get executed first in such a case connection pools make sense. This package also supports cluster mode connections.
+
+Working of this package is simple it exposes an interface [i-redis-client-pool](https://github.com/LRagji/redis-abstraction/blob/main/source/i-redis-client-pool.ts) which has following methods 
+1. `acquire(token: string): Promise<void>` : Responsibile to acquire a connection to redis server in-reference to the unique token provided.
+2. `run(token: string, commandArgs: string[]): Promise<any>` : Responsible to run a redis command in-reference to the unique token acquired before.
+3. `release(token: string): Promise<void>` : Responsible to release the acquired connection back into connection pool in-reference to the unique token acquired before.
+
+There are some more supporting methods as given below:
+* `generateUniqueToken(prefix: string): string;` : Generates a unique token for a given prefix, this can be then used to acquire and release connections from pool.
+* `pipeline(token: string, commands: string[][], transaction: boolean): Promise<any>` : Executes a set of commands together either in transaction or just close to one another.
+* `script(token: string, filename: string, keys: string[], args: string[]): Promise<any>` : Registers and executes a lua script.
+
+Currently this package only supports ioredis as underneath client library, but in future it may expand to other libraries as well.
+
+## Getting Started
+
+1. Install using `npm -i redis-abstraction`
+2. Require in your project. `const { IORedisClientPool } = require('redis-abstraction');` or `import { IORedisClientPool } from 'redis-abstraction';`
+3. Run redis on local docker if required. `docker run --name streamz -p 6379:6379 -itd --rm redis:latest`
+4. All done, Start using it!!.
+
+## Examples/Code snippets
+
+1. Please find example code in [examples](https://github.com/LRagji/redis-abstraction/blob/main/examples) folder
+2. Please find example code usage in  [unit tests](https://github.com/LRagji/redis-abstraction/blob/main/tests/specs-ioredis-client-pool.ts)
+
+
+### Non Cluster Initialization
+
+```javascript
+//Define the redis connection string
+const singleNodeRedisConnectionString = 'rediss://redis.my-service.com';
+//Create a injector function for creating redis connection instance.
+const connectionInjector = () => IORedisClientPool.IORedisClientClusterFactory([singleNodeRedisConnectionString]);
+//Initialize the pool
+const pool = new IORedisClientPool(connectionInjector);
+
+//Pass it around in the application.
+main(pool)
+    .finally(async () => {
+        //Remember to call shutdown which closes all connections in pool, else node.js process will not exit.
+        await pool.shutdown()
+    })
+```
+### Clustered Initialization
+
+```javascript
+//Define the redis connection string
+const clusteredRedisConnectionStringPrimary = 'rediss://redis.my-service.com';
+const clusteredRedisConnectionStringSecondary = 'rediss://redis.my-service.com' || clusteredRedisConnectionStringPrimary; //Secondary is optional if not present pass in primary connection.
+//Create a injector function for creating redis connection instance.
+const connectionInjector = () => IORedisClientPool.IORedisClientClusterFactory([clusteredRedisConnectionStringPrimary,clusteredRedisConnectionStringSecondary]);//Passing more than one connection string indicates its a cluster setup.
+//Initialize the pool
+const pool = new IORedisClientPool(connectionInjector);
+
+//Pass it around in the application.
+main(pool)
+    .finally(async () => {
+        //Remember to call shutdown which closes all connections in pool, else node.js process will not exit.
+        await pool.shutdown()
+    })
+```
+
+## Usage
+
+```javascript
+    //Generates a unique token
+    const token = pool.generateUniqueToken('Test');
+    try {
+        //Acquire idle connection from the pool or create a fresh one if entire pool connections are busy
+        await pool.acquire(token);
+        //Execute the command on the acquired connection
+        await pool.run(token, ['set', 'key', 'value']);
+    }
+    finally {
+        //Release of connection is important as it makes it available for others to acquire.
+        await pool.release(token);
+    }
+```
+
+## Built with
+
+1. Authors :heart: for Open Source.
+
+
+## Contributions
+
+1. New ideas/techniques are welcomed.
+2. Raise a Pull Request.
+
+## License
+
+This project is contrubution to public domain and completely free for use, view [LICENSE.md](/license.md) file for details.

package/i-redis-client-pool.d.ts

@@ -0,0 +1,47 @@
+/**
+* Interface used to abstract a redis client/connection for this package
+*/
+export interface IRedisClientPool {
+    /**
+     * This method acquires a redis client instance from a pool of redis clients with token as an identifier/handle.
+     * @param token A unique string used to acquire a redis client instance against. Treat this as redis client handle.
+     */
+    acquire(token: string): Promise<void>;
+    /**
+     * This method releases the acquired redis client back into the pool.
+     * @param token A unique string used when acquiring client via {@link acquire} method
+     */
+    release(token: string): Promise<void>;
+    /**
+     * Signals a dispose method to the pool stating no more clients will be needed, donot call any methods post calling shutdown.
+     */
+    shutdown(): Promise<void>;
+    /**
+     * Executes a single command on acquired connection.
+     * @param token token string which was used to acquire.
+     * @param commandArgs Array of strings including commands and arguments Eg:["set","key","value"]
+     * @returns Promise of any type.
+     */
+    run(token: string, commandArgs: string[]): Promise<any>;
+    /**
+     * This method is used to execute a set of commands in one go sequentially on redis side.
+     * @param token token string which was used to acquire.
+     * @param commands Array of array of strings including multiple commands and arguments that needs to be executed in one trip to the server sequentially. Eg:[["set","key","value"],["get","key"]]
+     * @param transaction Boolean value to indicate if to run the commands as transaction on redis or just run them close to each other.
+     * @returns Promise of all results in the commands(any type).
+     */
+    pipeline(token: string, commands: string[][], transaction: boolean): Promise<any>;
+    /**
+     * This method is used to execute a lua script on redis connection.
+     * @param token token string which was used to acquire.
+     * @param filePath Full file path of the lua script to be executed Eg: path.join(__dirname, "script.lua")
+     * @param keys Array of strings, Keys to be passsed to the script.
+     * @param args Array of strings, Arguments to be passed to the script.
+     */
+    script(token: string, filePath: string, keys: string[], args: string[]): Promise<any>;
+    /**
+     * This method should provide unique token for a given prefix.
+     * @param prefix An identity string for token to prepend.
+     */
+    generateUniqueToken(prefix: string): string;
+}

package/i-redis-client-pool.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/index.d.ts

@@ -0,0 +1,2 @@
+export { IRedisClientPool } from './i-redis-client-pool';
+export { IORedisClientPool } from './ioredis-client-pool';

package/index.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IORedisClientPool = void 0;
+var ioredis_client_pool_1 = require("./ioredis-client-pool");
+Object.defineProperty(exports, "IORedisClientPool", { enumerable: true, get: function () { return ioredis_client_pool_1.IORedisClientPool; } });

package/ioredis-client-pool.d.ts

@@ -0,0 +1,33 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { IRedisClientPool } from "./i-redis-client-pool";
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import Redis, { Cluster } from 'ioredis';
+export declare type RedisConnection = Cluster | Redis;
+export declare class IORedisClientPool implements IRedisClientPool {
+    private readonly nodeFSModule;
+    private readonly nodeCryptoModule;
+    private poolRedisClients;
+    private activeRedisClients;
+    private filenameToCommand;
+    private redisConnectionCreator;
+    private idlePoolSize;
+    private totalConnectionCounter;
+    constructor(redisConnectionCreator: () => RedisConnection, idlePoolSize?: number, nodeFSModule?: typeof fs, nodeCryptoModule?: typeof crypto);
+    generateUniqueToken(prefix: string): string;
+    shutdown(): Promise<void>;
+    acquire(token: string): Promise<void>;
+    release(token: string): Promise<void>;
+    run(token: string, commandArgs: any): Promise<unknown>;
+    pipeline(token: string, commands: string[][], transaction?: boolean): Promise<unknown[] | undefined>;
+    script(token: string, filePath: string, keys: string[], args: any[]): Promise<any>;
+    info(): {
+        "Idle Size": number;
+        "Current Active": number;
+        "Pooled Connection": number;
+        "Peak Connections": number;
+    };
+    private MD5Hash;
+    static IORedisClientClusterFactory(connectionDetails: string[], instanceInjection?: <T>(c: new (...args: any) => T, args: any[]) => T): RedisConnection;
+}

package/ioredis-client-pool.js

@@ -0,0 +1,171 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IORedisClientPool = void 0;
+const node_crypto_1 = __importDefault(require("node:crypto"));
+const node_fs_1 = __importDefault(require("node:fs"));
+const ioredis_1 = __importStar(require("ioredis"));
+const utils_1 = require("ioredis/built/utils");
+function createInstance(c, args = []) { return new c(...args); }
+class IORedisClientPool {
+    constructor(redisConnectionCreator, idlePoolSize = 6, nodeFSModule = node_fs_1.default, nodeCryptoModule = node_crypto_1.default) {
+        this.nodeFSModule = nodeFSModule;
+        this.nodeCryptoModule = nodeCryptoModule;
+        this.filenameToCommand = new Map();
+        this.totalConnectionCounter = 0;
+        this.poolRedisClients = Array.from({ length: idlePoolSize }, (_) => redisConnectionCreator());
+        this.totalConnectionCounter += idlePoolSize;
+        this.activeRedisClients = new Map();
+        this.redisConnectionCreator = redisConnectionCreator;
+        this.idlePoolSize = idlePoolSize;
+    }
+    generateUniqueToken(prefix) {
+        return `${prefix}-${this.nodeCryptoModule.randomUUID()}`;
+    }
+    shutdown() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const waitHandles = [...this.poolRedisClients, ...Array.from(this.activeRedisClients.values())]
+                .map((_) => __awaiter(this, void 0, void 0, function* () { yield _.quit(); _.disconnect(); }));
+            yield Promise.allSettled(waitHandles);
+            this.poolRedisClients = [];
+            this.activeRedisClients.clear();
+            this.totalConnectionCounter = 0;
+        });
+    }
+    acquire(token) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.activeRedisClients.has(token)) {
+                const availableClient = this.poolRedisClients.pop() || (() => { this.totalConnectionCounter += 1; return this.redisConnectionCreator(); })();
+                this.activeRedisClients.set(token, availableClient);
+            }
+        });
+    }
+    release(token) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const releasedClient = this.activeRedisClients.get(token);
+            if (releasedClient == undefined) {
+                return;
+            }
+            this.activeRedisClients.delete(token);
+            if (this.poolRedisClients.length < this.idlePoolSize) {
+                this.poolRedisClients.push(releasedClient);
+            }
+            else {
+                yield releasedClient.quit();
+                releasedClient.disconnect();
+            }
+        });
+    }
+    run(token, commandArgs) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const redisClient = this.activeRedisClients.get(token);
+            if (redisClient == undefined) {
+                throw new Error("Please acquire a client with proper token");
+            }
+            return yield redisClient.call(commandArgs.shift(), ...commandArgs);
+        });
+    }
+    pipeline(token, commands, transaction = true) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const redisClient = this.activeRedisClients.get(token);
+            if (redisClient == undefined) {
+                throw new Error("Please acquire a client with proper token");
+            }
+            const result = transaction === true ? yield redisClient.multi(commands).exec() : yield redisClient.pipeline(commands).exec();
+            return result === null || result === void 0 ? void 0 : result.map(r => {
+                let err = r[0];
+                if (err != null) {
+                    throw err;
+                }
+                return r[1];
+            });
+        });
+    }
+    script(token, filePath, keys, args) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const redisClient = this.activeRedisClients.get(token);
+            if (redisClient == undefined) {
+                throw new Error("Please acquire a client with proper token");
+            }
+            let command = this.filenameToCommand.get(filePath);
+            // @ts-ignore
+            if (command == null || redisClient[command] == null) {
+                const contents = yield this.nodeFSModule.promises.readFile(filePath, { encoding: "utf-8" });
+                command = this.MD5Hash(contents);
+                redisClient.defineCommand(command, { lua: contents });
+                this.filenameToCommand.set(filePath, command);
+            }
+            // @ts-ignore
+            return yield redisClient[command](keys.length, keys, args);
+        });
+    }
+    info() {
+        const returnObj = {
+            "Idle Size": this.idlePoolSize,
+            "Current Active": this.activeRedisClients.size,
+            "Pooled Connection": this.poolRedisClients.length,
+            "Peak Connections": this.totalConnectionCounter
+        };
+        this.totalConnectionCounter = 0;
+        return returnObj;
+    }
+    MD5Hash(value) {
+        return this.nodeCryptoModule.createHash('md5').update(value).digest('hex');
+    }
+    static IORedisClientClusterFactory(connectionDetails, instanceInjection = createInstance) {
+        const distinctConnections = new Set(connectionDetails);
+        if (distinctConnections.size === 0) {
+            throw new Error("Inncorrect or Invalid Connection details, cannot be empty");
+        }
+        if (connectionDetails.length > distinctConnections.size || distinctConnections.size > 1) {
+            const parsedRedisURl = (0, utils_1.parseURL)(connectionDetails[0]); //Assuming all have same password(they should have finally its a cluster)
+            const awsElasticCacheOptions = {
+                dnsLookup: (address, callback) => callback(null, address),
+                redisOptions: {
+                    tls: connectionDetails[0].startsWith("rediss:") == true ? {} : undefined,
+                    password: parsedRedisURl.password,
+                    maxRedirections: 32
+                },
+            };
+            return instanceInjection(ioredis_1.Cluster, [Array.from(distinctConnections.values()), awsElasticCacheOptions]);
+        }
+        else {
+            return instanceInjection(ioredis_1.default, [connectionDetails[0]]);
+        }
+    }
+}
+exports.IORedisClientPool = IORedisClientPool;

package/license.md

@@ -0,0 +1,9 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means.
+
+In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to https://unlicense.org

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "redis-abstraction",
+  "version": "0.0.2",
+  "description": "A Redis client pool with abstraction to different Redis libraries.",
+  "main": "index.js",
+  "scripts": {
+    "clean-build": "run-script-os",
+    "copy-files": "run-script-os",
+    "copy-files:linux": "cp ./README.md ./package.json ./package-lock.json ./license.md ./dist/",
+    "copy-files:macos": "cp ./README.md ./package.json ./package-lock.json ./license.md ./dist/",
+    "copy-files:windows": "for %I in (.\\README.md .\\package.json .\\package-lock.json .\\license.md) do copy %I .\\dist\\",
+    "clean-build:macos": "rm -rf ./dist/",
+    "clean-build:linux": "rm -rf ./dist/",
+    "clean-build:windows": "rmdir /s /q .\\dist\\",
+    "test-run": "nyc --reporter=html --reporter=text mocha -r ts-node/register ./tests/**/*.ts",
+    "test": "npm run build && npm run test-run",
+    "build": "(npm run clean-build || node -v) && tsc && npm run copy-files && npm run docs",
+    "redisstop": "docker stop TestCentralStore && ping 127.0.0.1 -c 3",
+    "redisstart": "(npm run redisstop || docker -v ) && docker run --name TestCentralStore -v ${PWD}:\"/var/lib/luatest\" -p 6379:6379 -itd --rm redis:latest",
+    "redisView": "docker run -v redisinsight:/db -p 8001:8001 -itd --rm redislabs/redisinsight:latest",
+    "push": "npm whoami && npm version patch && npm test && cd ./dist && npm publish && cd .. && git push --tags",
+    "docs": "typedoc"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/LRagji/redis-abstraction.git"
+  },
+  "author": "Laukik",
+  "license": "SEE LICENSE IN license.md",
+  "bugs": {
+    "url": "https://github.com/LRagji/redis-abstraction/issues"
+  },
+  "homepage": "https://github.com/LRagji/redis-abstraction#readme",
+  "keywords": [
+    "cluster",
+    "redis",
+    "client",
+    "abstraction",
+    "Laukik"
+  ],
+  "devDependencies": {
+    "@types/mocha": "^9.1.1",
+    "@types/sinon": "^10.0.15",
+    "cross-env": "^7.0.3",
+    "mocha": "^9.2.2",
+    "nyc": "^15.1.0",
+    "run-script-os": "^1.1.6",
+    "sinon": "^15.2.0",
+    "ts-node": "^10.7.0",
+    "typedoc": "^0.22.14",
+    "typescript": "^4.6.3"
+  },
+  "dependencies": {
+    "ioredis": "^5.3.2"
+  }
+}