pixl-server-storage 4.1.0 → 4.2.0

package/engines/Redis.js

@@ -9,6 +9,8 @@ const Class = require("pixl-class");
 const Component = require("pixl-server/component");
 const Redis = require('ioredis');
 const Tools = require("pixl-tools");
+const Perf = require("pixl-perf");
+const Cache = require("pixl-cache");
 
 module.exports = Class.create({
 	
@@ -47,6 +49,23 @@ module.exports = Class.create({
 		this.keyTemplate = (r_config.keyTemplate || '').replace(/^\//, '').replace(/\/$/, '');
 		delete r_config.keyTemplate;
 		
+		// optional LRU cache
+		this.cache = null;
+		var cache_opts = r_config.cache;
+		if (cache_opts && cache_opts.enabled) {
+			this.logDebug(3, "Setting up LRU cache", cache_opts);
+			this.cache = new Cache( Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) );
+			this.cache.on('expire', function(item, reason) {
+				self.logDebug(9, "Expiring LRU cache object: " + item.key + " due to: " + reason, {
+					key: item.key,
+					reason: reason,
+					totalCount: self.cache.count,
+					totalBytes: self.cache.bytes
+				});
+			});
+		}
+		delete r_config.cache;
+
 		if (!r_config.username.length) delete r_config.username;
 		if (!r_config.password.length) delete r_config.password;
 		
@@ -93,13 +112,14 @@ module.exports = Class.create({
 		// store key+value in Redis
 		var self = this;
 		key = this.prepKey(key);
+		var is_binary = this.storage.isBinaryKey(key);
 		
-		if (this.storage.isBinaryKey(key)) {
+		if (is_binary) {
 			this.logDebug(9, "Storing Redis Binary Object: " + key, '' + value.length + ' bytes');
 		}
 		else {
 			this.logDebug(9, "Storing Redis JSON Object: " + key, this.debugLevel(10) ? value : null);
-			value = JSON.stringify( value );
+			value = Buffer.from( JSON.stringify( value ) );
 		}
 		
 		this.redis.set( key, value, function(err) {
@@ -109,6 +129,11 @@ module.exports = Class.create({
 			}
 			else self.logDebug(9, "Store complete: " + key);
 			
+			// possibly cache in LRU
+			if (!err && self.cache && !is_binary) {
+				self.cache.set( key, value, { date: Tools.timeNow(true) } );
+			}
+
 			if (callback) callback(err);
 		} );
 	},
@@ -138,7 +163,30 @@ module.exports = Class.create({
 		// The Redis API has no way to head / ping an object.
 		// So, we have to do this the RAM-hard way...
 		
-		this.redis.get( key, function(err, data) {
+		// check cache first
+		if (this.cache && this.cache.has(key)) {
+			var item = this.cache.getMeta(key);
+
+			if (item.value.length == 0) {
+				process.nextTick( function() {
+					var err = new Error("Failed to head key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+					callback( err );
+				} );
+			}
+			else {
+				process.nextTick( function() {
+					self.logDebug(9, "Cached head complete: " + key);
+					callback( null, {
+						mod: item.date,
+						len: item.value.length
+					} );
+				} );
+			}
+			return;
+		} // cache
+
+		this.redis.getBuffer( key, function(err, data) {
 			if (err) {
 				// an actual error
 				err.message = "Failed to head key: " + key + ": " + err;
@@ -150,7 +198,12 @@ module.exports = Class.create({
 				// always use "NoSuchKey" in error code
 				var err = new Error("Failed to head key: " + key + ": Not found");
 				err.code = "NoSuchKey";
-				
+
+				if (self.cache && !self.storage.isBinaryKey(key)) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+				}
+
 				callback( err, null );
 			}
 			else {
@@ -163,44 +216,79 @@ module.exports = Class.create({
 		// fetch Redis value given key
 		var self = this;
 		key = this.prepKey(key);
+		var is_binary = this.storage.isBinaryKey(key);
 		
 		this.logDebug(9, "Fetching Redis Object: " + key);
 		
-		var func = this.storage.isBinaryKey(key) ? 'getBuffer' : 'get';
-		this.redis[func]( key, function(err, result) {
-			if (!result) {
-				if (err) {
-					// an actual error
-					err.message = "Failed to fetch key: " + key + ": " + err;
-					self.logError('redis', ''+err);
-					callback( err, null );
-				}
-				else {
-					// record not found
-					// always use "NoSuchKey" in error code
+		// check cache first
+		if (this.cache && !is_binary && this.cache.has(key)) {
+			var item = this.cache.getMeta(key);
+
+			if (item.value.length == 0) {
+				process.nextTick( function() {
 					var err = new Error("Failed to fetch key: " + key + ": Not found");
 					err.code = "NoSuchKey";
-					
-					callback( err, null );
-				}
+					callback( err );
+				} );
 			}
 			else {
-				if (self.storage.isBinaryKey(key)) {
-					self.logDebug(9, "Binary fetch complete: " + key, '' + result.length + ' bytes');
-				}
-				else {
-					try { result = JSON.parse( result.toString() ); }
+				process.nextTick( function() {
+					var data = null;
+					try { data = JSON.parse( item.value.toString() ); }
 					catch (err) {
 						self.logError('redis', "Failed to parse JSON record: " + key + ": " + err);
 						callback( err, null );
 						return;
 					}
-					self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? result : null);
+					self.logDebug(9, "Cached JSON fetch complete: " + key, self.debugLevel(10) ? data : null);
+
+					callback( null, data );
+				} );
+			}
+			return;
+		} // cache
+
+		this.redis.getBuffer( key, function(err, result) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('redis', ''+err);
+				return callback( err, null );
+			}
+			else if (!result || !result.length) {
+				// record not found
+				// always use "NoSuchKey" in error code
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				
+				if (self.cache && !is_binary) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
 				}
 				
-				callback( null, result );
+				return callback( err, null );
 			}
-		} );
+			
+			if (is_binary) {
+				self.logDebug(9, "Binary fetch complete: " + key, '' + result.length + ' bytes');
+			}
+			else {
+				// possibly cache in LRU
+				if (self.cache) {
+					self.cache.set( key, result, { date: Tools.timeNow(true) } );
+				}
+				
+				try { result = JSON.parse( result.toString() ); }
+				catch (err) {
+					self.logError('redis', "Failed to parse JSON record: " + key + ": " + err);
+					callback( err, null );
+					return;
+				}
+				self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? result : null);
+			}
+			
+			callback( null, result );
+		} ); // redis.getBuffer
 	},
 	
 	getBuffer: function(key, callback) {
@@ -303,6 +391,7 @@ module.exports = Class.create({
 		// delete Redis key given key
 		var self = this;
 		key = this.prepKey(key);
+		var is_binary = this.storage.isBinaryKey(key);
 		
 		this.logDebug(9, "Deleting Redis Object: " + key);
 		
@@ -310,21 +399,166 @@ module.exports = Class.create({
 			if (!err && !deleted) {
 				err = new Error("Failed to fetch key: " + key + ": Not found");
 				err.code = "NoSuchKey";
+
+				if (self.cache && !is_binary) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+				}
 			}
 			if (err) {
 				self.logError('redis', "Failed to delete object: " + key + ": " + err);
 			}
-			else self.logDebug(9, "Delete complete: " + key);
+			else {
+				self.logDebug(9, "Delete complete: " + key);
+
+				// possibly "delete" from LRU cache as well
+				if (self.cache && !is_binary) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+				}
+			}
 			
 			callback(err);
 		} );
 	},
 	
+	commitTransaction: function(trans, callback) {
+		// commit all transaction actions using a native Redis MULTI/EXEC transaction
+		var self = this;
+		var write_keys = [];
+		var write_values = [];
+		var write_modified = [];
+		var delete_keys = [];
+		var bad_state = null;
+		var multi = this.redis.multi();
+		
+		this.logDebug(5, "Beginning transaction commit: " + trans.id, {
+			path: trans.path,
+			actions: Tools.numKeys(trans.keys)
+		});
+		
+		Object.keys(trans.keys).forEach( function(key) {
+			var record_state = trans.keys[key];
+			var norm_key = self.storage.normalizeKey(key);
+			
+			if (record_state == 'W') {
+				// Collect writes into parallel arrays so we can update the cache later
+				// without stringifying and Buffer-wrapping JSON a second time.
+				var item = trans.values[key];
+				var redis_key = self.prepKey(norm_key);
+				var value = Buffer.from( JSON.stringify( item.data ) );
+				
+				write_keys.push( redis_key );
+				write_values.push( value );
+				write_modified.push( item.mod );
+				
+				multi.set( redis_key, value );
+			}
+			else if (record_state == 'D') {
+				// Queue deletes into the same Redis transaction.
+				var redis_key = self.prepKey(norm_key);
+				delete_keys.push( redis_key );
+				
+				multi.del( redis_key );
+			}
+			else {
+				bad_state = new Error("Unknown transaction record state: " + record_state + ": " + key);
+			}
+		});
+		
+		if (bad_state) return callback(bad_state);
+		
+		multi.exec( function(err, results) {
+			if (err) {
+				// With MULTI/EXEC over the network, a top-level exec error may leave
+				// the actual Redis outcome unknowable, so treat it as fatal.
+				err.fatal = true;
+				self.logError('redis', "Transaction commit failed: " + err);
+				return callback(err);
+			}
+			
+			// Redis can return per-command errors from EXEC while still applying
+			// other commands.  That is a partial commit, so it must be fatal.
+			var cmd_err = null;
+			if (results) {
+				results.forEach( function(item) {
+					if (item && item[0] && !cmd_err) cmd_err = item[0];
+				});
+			}
+			if (cmd_err) {
+				cmd_err.fatal = true;
+				self.logError('redis', "Transaction command failed: " + cmd_err);
+				return callback(cmd_err);
+			}
+			
+			// commit succeeded, so now it is safe to update the engine LRU cache
+			if (self.cache) {
+				write_keys.forEach( function(key, idx) {
+					if (!self.storage.isBinaryKey(key)) {
+						self.cache.set( key, write_values[idx], { date: write_modified[idx] } );
+					}
+				});
+				
+				delete_keys.forEach( function(key) {
+					if (!self.storage.isBinaryKey(key)) {
+						// store 'empty' stub in cache
+						self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+					}
+				});
+			}
+			
+			self.logDebug(5, "Transaction commit complete: " + trans.id, {
+				path: trans.path
+			});
+			
+			callback();
+		} );
+	},
+	
+	unitTestCleanup: function(callback) {
+		// cleanup all unit test data
+		var self = this;
+		this.logDebug(3, "Cleaning up Redis unit test database with FLUSHDB");
+		
+		return this.redis.flushdb( function(err) {
+			if (err) {
+				self.logError('redis', "Failed to cleanup Redis unit test database: " + err);
+				return callback(err);
+			}
+			
+			if (self.cache) self.cache.clear();
+			callback();
+		} );
+	},
+	
 	runMaintenance: function(callback) {
 		// run daily maintenance
 		callback();
 	},
 	
+	optimize: function(callback) {
+		// Redis does its own memory management, so there is nothing to run here.
+		var perf = new Perf();
+		perf.setScale(1);
+		perf.begin();
+		perf.begin('noop');
+		perf.end('noop');
+		perf.end();
+		
+		process.nextTick( callback, null, {
+			engine: this.__name,
+			optimized: false,
+			perf: perf.metrics(),
+			operations: [
+				{
+					name: 'noop',
+					ok: true,
+					message: "Redis does not require storage optimization."
+				}
+			]
+		} );
+	},
+	
 	shutdown: function(callback) {
 		// shutdown storage
 		this.logDebug(2, "Shutting down Redis");

package/engines/RedisCluster.js

@@ -9,6 +9,8 @@ const Class = require("pixl-class");
 const Component = require("pixl-server/component");
 const Redis = require('ioredis');
 const Tools = require("pixl-tools");
+const Perf = require("pixl-perf");
+const Cache = require("pixl-cache");
 
 module.exports = Class.create({
 	
@@ -49,6 +51,22 @@ module.exports = Class.create({
 		
 		this.keyTemplate = (r_config.keyTemplate || '').replace(/^\//, '').replace(/\/$/, '');
 		
+		// optional LRU cache
+		this.cache = null;
+		var cache_opts = r_config.cache;
+		if (cache_opts && cache_opts.enabled) {
+			this.logDebug(3, "Setting up LRU cache", cache_opts);
+			this.cache = new Cache( Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) );
+			this.cache.on('expire', function(item, reason) {
+				self.logDebug(9, "Expiring LRU cache object: " + item.key + " due to: " + reason, {
+					key: item.key,
+					reason: reason,
+					totalCount: self.cache.count,
+					totalBytes: self.cache.bytes
+				});
+			});
+		}
+
 		r_config.clusterOpts.clusterRetryStrategy = function(attempts) {
 			if (attempts > r_config.connectRetries) return false;
 			return attempts;
@@ -107,13 +125,14 @@ module.exports = Class.create({
 		// store key+value in Redis
 		var self = this;
 		key = this.prepKey(key);
+		var is_binary = this.storage.isBinaryKey(key);
 		
-		if (this.storage.isBinaryKey(key)) {
+		if (is_binary) {
 			this.logDebug(9, "Storing Redis Binary Object: " + key, '' + value.length + ' bytes');
 		}
 		else {
 			this.logDebug(9, "Storing Redis JSON Object: " + key, this.debugLevel(10) ? value : null);
-			value = JSON.stringify( value );
+			value = Buffer.from( JSON.stringify( value ) );
 		}
 		
 		this.redis.set( key, value, function(err) {
@@ -123,6 +142,11 @@ module.exports = Class.create({
 			}
 			else self.logDebug(9, "Store complete: " + key);
 			
+			// possibly cache in LRU
+			if (!err && self.cache && !is_binary) {
+				self.cache.set( key, value, { date: Tools.timeNow(true) } );
+			}
+
 			if (callback) callback(err);
 		} );
 	},
@@ -152,7 +176,30 @@ module.exports = Class.create({
 		// The Redis API has no way to head / ping an object.
 		// So, we have to do this the RAM-hard way...
 		
-		this.redis.get( key, function(err, data) {
+		// check cache first
+		if (this.cache && this.cache.has(key)) {
+			var item = this.cache.getMeta(key);
+
+			if (item.value.length == 0) {
+				process.nextTick( function() {
+					var err = new Error("Failed to head key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+					callback( err );
+				} );
+			}
+			else {
+				process.nextTick( function() {
+					self.logDebug(9, "Cached head complete: " + key);
+					callback( null, {
+						mod: item.date,
+						len: item.value.length
+					} );
+				} );
+			}
+			return;
+		} // cache
+
+		this.redis.getBuffer( key, function(err, data) {
 			if (err) {
 				// an actual error
 				err.message = "Failed to head key: " + key + ": " + err;
@@ -164,7 +211,12 @@ module.exports = Class.create({
 				// always use "NoSuchKey" in error code
 				var err = new Error("Failed to head key: " + key + ": Not found");
 				err.code = "NoSuchKey";
-				
+
+				if (self.cache && !self.storage.isBinaryKey(key)) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+				}
+
 				callback( err, null );
 			}
 			else {
@@ -177,44 +229,79 @@ module.exports = Class.create({
 		// fetch Redis value given key
 		var self = this;
 		key = this.prepKey(key);
+		var is_binary = this.storage.isBinaryKey(key);
 		
 		this.logDebug(9, "Fetching Redis Object: " + key);
 		
-		var func = this.storage.isBinaryKey(key) ? 'getBuffer' : 'get';
-		this.redis[func]( key, function(err, result) {
-			if (!result) {
-				if (err) {
-					// an actual error
-					err.message = "Failed to fetch key: " + key + ": " + err;
-					self.logError('redis', ''+err);
-					callback( err, null );
-				}
-				else {
-					// record not found
-					// always use "NoSuchKey" in error code
+		// check cache first
+		if (this.cache && !is_binary && this.cache.has(key)) {
+			var item = this.cache.getMeta(key);
+			
+			if (item.value.length == 0) {
+				process.nextTick( function() {
 					var err = new Error("Failed to fetch key: " + key + ": Not found");
 					err.code = "NoSuchKey";
-					
-					callback( err, null );
-				}
+					callback( err );
+				} );
 			}
 			else {
-				if (self.storage.isBinaryKey(key)) {
-					self.logDebug(9, "Binary fetch complete: " + key, '' + result.length + ' bytes');
-				}
-				else {
-					try { result = JSON.parse( result.toString() ); }
+				process.nextTick( function() {
+					var data = null;
+					try { data = JSON.parse( item.value.toString() ); }
 					catch (err) {
 						self.logError('redis', "Failed to parse JSON record: " + key + ": " + err);
 						callback( err, null );
 						return;
 					}
-					self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? result : null);
+					self.logDebug(9, "Cached JSON fetch complete: " + key, self.debugLevel(10) ? data : null);
+					
+					callback( null, data );
+				} );
+			}
+			return;
+		} // cache
+		
+		this.redis.getBuffer( key, function(err, result) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('redis', ''+err);
+				return callback( err, null );
+			}
+			else if (!result || !result.length) {
+				// record not found
+				// always use "NoSuchKey" in error code
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+
+				if (self.cache && !is_binary) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
 				}
 				
-				callback( null, result );
+				return callback( err, null );
 			}
-		} );
+			
+			if (is_binary) {
+				self.logDebug(9, "Binary fetch complete: " + key, '' + result.length + ' bytes');
+			}
+			else {
+				// possibly cache in LRU
+				if (self.cache) {
+					self.cache.set( key, result, { date: Tools.timeNow(true) } );
+				}
+				
+				try { result = JSON.parse( result.toString() ); }
+				catch (err) {
+					self.logError('redis', "Failed to parse JSON record: " + key + ": " + err);
+					callback( err, null );
+					return;
+				}
+				self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? result : null);
+			}
+			
+			callback( null, result );
+		} ); // redis.getBuffer
 	},
 	
 	getBuffer: function(key, callback) {
@@ -317,6 +404,7 @@ module.exports = Class.create({
 		// delete Redis key given key
 		var self = this;
 		key = this.prepKey(key);
+		var is_binary = this.storage.isBinaryKey(key);
 		
 		this.logDebug(9, "Deleting Redis Object: " + key);
 		
@@ -324,11 +412,24 @@ module.exports = Class.create({
 			if (!err && !deleted) {
 				err = new Error("Failed to fetch key: " + key + ": Not found");
 				err.code = "NoSuchKey";
+
+				if (self.cache && !is_binary) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+				}
 			}
 			if (err) {
 				self.logError('redis', "Failed to delete object: " + key + ": " + err);
 			}
-			else self.logDebug(9, "Delete complete: " + key);
+			else {
+				self.logDebug(9, "Delete complete: " + key);
+
+				// possibly "delete" from LRU cache as well
+				if (self.cache && !is_binary) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+				}
+			}
 			
 			callback(err);
 		} );
@@ -339,6 +440,29 @@ module.exports = Class.create({
 		callback();
 	},
 	
+	optimize: function(callback) {
+		// Redis Cluster does its own memory management, so there is nothing to run here.
+		var perf = new Perf();
+		perf.setScale(1);
+		perf.begin();
+		perf.begin('noop');
+		perf.end('noop');
+		perf.end();
+		
+		process.nextTick( callback, null, {
+			engine: this.__name,
+			optimized: false,
+			perf: perf.metrics(),
+			operations: [
+				{
+					name: 'noop',
+					ok: true,
+					message: "RedisCluster does not require storage optimization."
+				}
+			]
+		} );
+	},
+	
 	shutdown: function(callback) {
 		// shutdown storage
 		this.logDebug(2, "Shutting down Redis");