pixl-server-storage 3.1.18 → 3.2.1

package/engines/Redis.js

@@ -1,14 +1,14 @@
 // Redis Storage Plugin
-// Copyright (c) 2015 - 2019 Joseph Huckaby
+// Copyright (c) 2015 - 2024 Joseph Huckaby
 // Released under the MIT License
 
-// Requires the 'redis' module from npm
-// npm install redis
+// Requires the 'ioredis' module from npm
+// npm install --save ioredis
 
-var Class = require("pixl-class");
-var Component = require("pixl-server/component");
-var Redis = require('redis');
-var Tools = require("pixl-tools");
+const Class = require("pixl-class");
+const Component = require("pixl-server/component");
+const Redis = require('ioredis');
+const Tools = require("pixl-tools");
 
 module.exports = Class.create({
 	
@@ -16,8 +16,14 @@ module.exports = Class.create({
 	__parent: Component,
 	
 	defaultConfig: {
-		host: "127.0.0.1",
+		
+		host: 'localhost',
 		port: 6379,
+		commandTimeout: 5000,
+		connectTimeout: 5000,
+		username: "",
+		password: "",
+		
 		keyPrefix: "",
 		keyTemplate: ""
 	},
@@ -25,10 +31,7 @@ module.exports = Class.create({
 	startup: function(callback) {
 		// setup Redis connection
 		var self = this;
-		
-		this.logDebug(2, "Setting up Redis", 
-			Tools.copyHashRemoveKeys( this.config.get(), { password:1 }) );
-		
+		this.logDebug(2, "Setting up Redis", this.config.get() );
 		this.setup(callback);
 	},
 	
@@ -39,35 +42,31 @@ module.exports = Class.create({
 		
 		this.keyPrefix = (r_config.keyPrefix || '').replace(/^\//, '');
 		if (this.keyPrefix && !this.keyPrefix.match(/\/$/)) this.keyPrefix += '/';
+		delete r_config.keyPrefix;
 		
 		this.keyTemplate = (r_config.keyTemplate || '').replace(/^\//, '').replace(/\/$/, '');
+		delete r_config.keyTemplate;
 		
-		r_config.return_buffers = true;
-		r_config.retry_strategy = function(opts) {
-			// simple backoff strategy
-			return Math.min(opts.attempt * 100, 3000);
-		};
+		if (!r_config.username.length) delete r_config.username;
+		if (!r_config.password.length) delete r_config.password;
 		
-		this.redis = Redis.createClient( Tools.copyHashRemoveKeys(r_config, { keyPrefix:1, keyTemplate:1 }) );
+		r_config.lazyConnect = true;
+		r_config.reconnectOnError = function(err) { return true; };
+		
+		this.redis = new Redis(r_config);
 		
 		this.redis.on('error', function(err) {
-			if (!self.storage.started) return callback(err);
+			if (!self.storage.started) {
+				return callback( new Error("Redis Startup Error: " + (err.message || err)) );
+			}
 			
 			// error after startup?  Just log it I guess
 			self.logError('redis', ''+err);
-		});
+		}); // error
 		
-		this.redis.on('connect', function() {
-			self.logDebug(3, "Redis connected successfully");
-			if (!self.storage.started) return callback();
-		});
-		
-		this.redis.on('reconnecting', function(opts) {
-			self.logDebug(3, "Redis is reconnecting", opts);
-		});
-		
-		this.redis.on('end', function() {
-			self.logDebug(3, "Redis disconnected");
+		this.redis.connect(function() {
+			self.logDebug(8, "Successfully connected to Redis");
+			callback();
 		});
 	},
 	
@@ -167,7 +166,8 @@ module.exports = Class.create({
 		
 		this.logDebug(9, "Fetching Redis Object: " + key);
 		
-		this.redis.get( key, function(err, result) {
+		var func = this.storage.isBinaryKey(key) ? 'getBuffer' : 'get';
+		this.redis[func]( key, function(err, result) {
 			if (!result) {
 				if (err) {
 					// an actual error
@@ -204,13 +204,13 @@ module.exports = Class.create({
 	},
 	
 	getBuffer: function(key, callback) {
-		// get buffer to record value given key
+		// fetch Redis buffer given key
 		var self = this;
 		key = this.prepKey(key);
 		
 		this.logDebug(9, "Fetching Redis Object: " + key);
 		
-		this.redis.get( key, function(err, result) {
+		this.redis.getBuffer( key, function(err, result) {
 			if (!result) {
 				if (err) {
 					// an actual error
@@ -329,8 +329,11 @@ module.exports = Class.create({
 	shutdown: function(callback) {
 		// shutdown storage
 		this.logDebug(2, "Shutting down Redis");
-		if (this.redis) this.redis.quit();
-		callback();
+		if (this.redis) {
+			this.redis.quit(callback);
+			this.redis = null;
+		}
+		else callback();
 	}
 	
 });

package/engines/RedisCluster.js

@@ -1,14 +1,13 @@
 // RedisCluster Storage Plugin
-// Copyright (c) 2015 - 2020 Joseph Huckaby
+// Copyright (c) 2015 - 2024 Joseph Huckaby
 // Released under the MIT License
 
-// Requires the 'ioredis' and 'ioredis-timeout' modules from npm
-// npm install --save ioredis ioredis-timeout
+// Requires the 'ioredis' module from npm
+// npm install --save ioredis
 
 const Class = require("pixl-class");
 const Component = require("pixl-server/component");
 const Redis = require('ioredis');
-const RedisTimeout = require('ioredis-timeout');
 const Tools = require("pixl-tools");
 
 module.exports = Class.create({
@@ -20,10 +19,13 @@ module.exports = Class.create({
 		
 		host: 'localhost',
 		port: 6379,
-		timeout: 1000,
 		connectRetries: 5,
 		clusterOpts: {
-			scaleReads: "master"
+			scaleReads: "master",
+			redisOptions: {
+				commandTimeout: 5000,
+				connectTimeout: 5000
+			}
 		},
 		
 		keyPrefix: "",
@@ -33,7 +35,7 @@ module.exports = Class.create({
 	startup: function(callback) {
 		// setup Redis connection
 		var self = this;
-		this.logDebug(2, "Setting up RedisClustr", this.config.get() );
+		this.logDebug(2, "Setting up RedisCluster", this.config.get() );
 		this.setup(callback);
 	},
 	
@@ -77,12 +79,7 @@ module.exports = Class.create({
 		}); // error
 		
 		this.redis.once('ready', function() {
-			self.logDebug(8, "Successfully connected to Redis");
-			
-			if (r_config.timeout) {
-				RedisTimeout(self.redis, parseInt(r_config.timeout));
-			}
-			
+			self.logDebug(8, "Successfully connected to Redis cluster");
 			callback();
 		});
 	},

package/engines/S3.js

@@ -2,6 +2,7 @@
 // Copyright (c) 2015 - 2022 Joseph Huckaby
 // Released under the MIT License
 
+var Path = require('path');
 var Class = require("pixl-class");
 var Component = require("pixl-server/component");
 var Tools = require("pixl-tools");
@@ -596,6 +597,93 @@ module.exports = Class.create({
 			} );
 	},
 	
+	list: function(opts, callback) {
+		// generate list of objects in S3 given prefix
+		// this repeatedly calls ListObjectsV2 for lists > 1000
+		// opts: { remotePath, filespec, filter }
+		// result: { files([{ key, size, mtime }, ...]), total_bytes }
+		let self = this;
+		let done = false;
+		let files = [];
+		let total_bytes = 0;
+		let num_calls = 0;
+		let now = Tools.timeNow(true);
+		
+		if (typeof(opts) == 'string') opts = { remotePath: opts };
+		if (!opts.remotePath) opts.remotePath = '';
+		if (!opts.filespec) opts.filespec = /.*/;
+		if (!opts.filter) opts.filter = function() { return true; };
+		
+		if (opts.older) {
+			// convert older to filter func with mtime
+			if (typeof(opts.older) == 'string') opts.older = Tools.getSecondsFromText( opts.older );
+			opts.filter = function(file) { return file.mtime <= now - opts.older; };
+		}
+		
+		let params = Tools.mergeHashes( this.s3Params || {}, opts.params || {} );
+		params.Prefix = this.keyPrefix + opts.remotePath;
+		params.MaxKeys = 1000;
+		
+		let PREFIX_RE = new RegExp('^' + Tools.escapeRegExp(this.keyPrefix));
+		
+		this.logDebug(8, "Listing S3 files with prefix: " + params.Prefix, opts);
+		let tracker = this.perf ? this.perf.begin('s3_list') : null;
+		
+		Tools.async.whilst(
+			function() { 
+				return !done; 
+			},
+			function(callback) {
+				self.logDebug(9, "Listing chunk", params);
+				
+				self.s3.send( new S3.ListObjectsV2Command(params) )
+					.then( function(data) {
+						let items = data.Contents || [];
+						for (let idx = 0, len = items.length; idx < len; idx++) {
+							let item = items[idx];
+							let key = item.Key.replace(PREFIX_RE, '');
+							let bytes = item.Size;
+							let mtime = item.LastModified.getTime() / 1000;
+							let file = { key: key, size: bytes, mtime: mtime };
+							
+							// optional filter and filespec
+							if (opts.filter(file) && Path.basename(key).match(opts.filespec)) {
+								total_bytes += bytes;
+								files.push(file);
+							}
+						}
+						
+						// check for end of key list
+						if (!data.IsTruncated || !items.length) done = true;
+						else {
+							// advance to next chunk
+							params.StartAfter = items[ items.length - 1 ].Key;
+						}
+						
+						num_calls++;
+						callback();
+					} )
+					.catch( function(err) {
+						callback( err );
+					} );
+			},
+			function(err) {
+				if (tracker) tracker.end();
+				if (err) return process.nextTick( function() { callback(err, null, null); });
+				
+				self.logDebug(9, "S3 listing complete (" + Tools.commify(files.length) + " objects, " + Tools.getTextFromBytes(total_bytes) + ")", {
+					prefix: params.Prefix,
+					count: files.length,
+					bytes: total_bytes,
+					calls: num_calls
+				});
+				
+				// break out of promise context
+				process.nextTick( function() { callback( null, files, total_bytes ); } );
+			}
+		); // whilst
+	},
+	
 	runMaintenance: function(callback) {
 		// run daily maintenance
 		callback();

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.1.18",
+	"version": "3.2.1",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",