pixl-server-storage 4.1.1 → 4.2.0

package/engines/Hybrid.js

@@ -4,7 +4,7 @@
 
 var Class = require("pixl-class");
 var Component = require("pixl-server/component");
-var Tools = require("pixl-tools");
+var Perf = require("pixl-perf");
 var async = require('async');
 
 module.exports = Class.create({
@@ -20,7 +20,8 @@ module.exports = Class.create({
 	startup: function(callback) {
 		// setup multiple sub-engines
 		this.logDebug(2, "Setting up hybrid engine", this.config.get() );
-		
+		var self = this;
+
 		var binaryClass = require( './' + this.config.get('binaryEngine') + '.js' );
 		this.binaryEngine = new binaryClass();
 		this.binaryEngine.storage = this.storage;
@@ -31,6 +32,17 @@ module.exports = Class.create({
 		this.docEngine.storage = this.storage;
 		this.docEngine.init( this.server, this.storage.config.getSub( this.config.get('docEngine') ) );
 		
+		// Native transactions only apply to document records.  Binary keys are
+		// already passed straight through by the outer transaction wrapper, so
+		// only expose this hook if the doc engine can atomically commit JSON.
+		delete this.commitTransaction;
+		if (this.docEngine.commitTransaction) {
+			this.logDebug(3, "Hybrid engine will use native doc engine transactions");
+			this.commitTransaction = function(trans, callback) {
+				self.docEngine.commitTransaction( trans, callback );
+			};
+		}
+
 		async.series([
 			this.binaryEngine.startup.bind(this.binaryEngine),
 			this.docEngine.startup.bind(this.docEngine)
@@ -115,6 +127,96 @@ module.exports = Class.create({
 		], callback );
 	},
 	
+	optimize: function(callback) {
+		// run optimization on child engines that support it
+		var self = this;
+		var perf = new Perf();
+		var results = [];
+		
+		this.logDebug(3, "Running hybrid optimization");
+		perf.setScale(1);
+		perf.begin();
+		
+		var optimizeEngine = function(label, engine, callback) {
+			var tracker = perf.begin(label);
+			
+			if (!engine || !engine.optimize) {
+				var child_perf = new Perf();
+				child_perf.setScale(1);
+				child_perf.begin();
+				child_perf.begin('noop');
+				child_perf.end('noop');
+				child_perf.end();
+				tracker.end();
+				
+				results.push({
+					role: label,
+					engine: engine ? engine.__name : '',
+					optimized: false,
+					perf: child_perf.metrics(),
+					operations: [
+						{
+							name: 'noop',
+							ok: true,
+							message: "No optimization is available for this engine."
+						}
+					]
+				});
+				return process.nextTick(callback);
+			}
+			
+			engine.optimize( function(err, report) {
+				tracker.end();
+				if (err) return callback(err);
+				if (!report) report = {};
+				report.role = label;
+				if (!report.engine) report.engine = engine.__name;
+				results.push(report);
+				callback();
+			} );
+		};
+		
+		async.series([
+			function(callback) {
+				optimizeEngine( 'binary', self.binaryEngine, callback );
+			},
+			function(callback) {
+				optimizeEngine( 'doc', self.docEngine, callback );
+			}
+		], function(err) {
+			if (err) return callback(err);
+			perf.end();
+			
+			var report = {
+				engine: self.__name,
+				optimized: results.some( function(item) { return !!item.optimized; } ),
+				perf: perf.metrics(),
+				engines: results
+			};
+			
+			report.operations = results.reduce( function(list, item) {
+				return list.concat( item.operations || [] );
+			}, [] );
+			
+			callback(null, report);
+		} );
+	},
+	
+	unitTestCleanup: function(callback) {
+		// cleanup all child engines that provide a unit test cleanup hook
+		var tasks = [];
+
+		if (this.binaryEngine && this.binaryEngine.unitTestCleanup) {
+			tasks.push( this.binaryEngine.unitTestCleanup.bind(this.binaryEngine) );
+		}
+		if (this.docEngine && this.docEngine.unitTestCleanup) {
+			tasks.push( this.docEngine.unitTestCleanup.bind(this.docEngine) );
+		}
+
+		if (!tasks.length) return process.nextTick(callback);
+		async.series( tasks, callback );
+	},
+
 	shutdown: function(callback) {
 		// shutdown storage
 		this.logDebug(2, "Shutting down hybrid system");

package/engines/Postgres.js

@@ -53,7 +53,9 @@ module.exports = Class.create({
 			get: `SELECT modified, value FROM ${pg_config.table} WHERE key = $key LIMIT 1`,
 			head: `SELECT modified, octet_length(value) AS len FROM ${pg_config.table} WHERE key = $key LIMIT 1`,
 			put: `INSERT INTO ${pg_config.table} (key, value, modified) VALUES ($key, $value, $now) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, modified = EXCLUDED.modified`,
-			delete: `DELETE FROM ${pg_config.table} WHERE key = $key`
+			delete: `DELETE FROM ${pg_config.table} WHERE key = $key`,
+			transPut: `INSERT INTO ${pg_config.table} (key, value, modified) SELECT * FROM unnest($keys::text[], $values::bytea[], $modified::bigint[]) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, modified = EXCLUDED.modified`,
+			transDelete: `DELETE FROM ${pg_config.table} WHERE key = ANY($keys::text[])`
 		};
 		
 		// optional LRU cache
@@ -98,9 +100,15 @@ module.exports = Class.create({
 	},
 	
 	query(sql, args, callback) {
+		// run query against the pool
+		this.queryWithClient( this.db, sql, args, callback );
+	},
+	
+	queryWithClient: function(client, sql, args, callback) {
 		// run and log db query, and error if applicable
 		// allow for named variables in sql, expand to numbered $ variables for pg
 		var self = this;
+		if (!args) args = {};
 		
 		// perform $ placeholder substitution to populate sargs array
 		var sargs = [];
@@ -111,7 +119,7 @@ module.exports = Class.create({
 			return value;
 		} );
 		
-		this.db.query( sql, sargs, function(err, res) {
+		client.query( sql, sargs, function(err, res) {
 			if (err) {
 				self.logError('db', "Failed to execute SQL: " + err, { sql, err });
 				return callback(err);
@@ -123,7 +131,7 @@ module.exports = Class.create({
 			});
 			
 			callback(null, res);
-		} ); // db.query
+		} ); // client.query
 	},
 	
 	prepKey: function(key) {
@@ -462,11 +470,231 @@ module.exports = Class.create({
 		} ); // query
 	},
 	
+	commitTransaction: function(trans, callback) {
+		// commit all transaction actions using a native Postgres transaction
+		// IMPORTANT: all queries must run on this one checked-out client
+		var self = this;
+		var did_begin = false;
+		var write_keys = [];
+		var write_values = [];
+		var write_modified = [];
+		var delete_keys = [];
+		var bad_state = null;
+		var query_step = '';
+		
+		this.logDebug(5, "Beginning transaction commit: " + trans.id, {
+			path: trans.path,
+			actions: Tools.numKeys(trans.keys)
+		});
+		
+		Object.keys(trans.keys).forEach( function(key) {
+			var record_state = trans.keys[key];
+			var norm_key = self.storage.normalizeKey(key);
+			
+			if (record_state == 'W') {
+				// Collect writes into parallel arrays for one bulk UPSERT below.
+				var item = trans.values[key];
+				write_keys.push( self.prepKey(norm_key) );
+				write_values.push( Buffer.from( JSON.stringify( item.data ) ) );
+				write_modified.push( item.mod );
+			}
+			else if (record_state == 'D') {
+				// Collect deletes into one bulk DELETE below.
+				delete_keys.push( self.prepKey(norm_key) );
+			}
+			else {
+				bad_state = new Error("Unknown transaction record state: " + record_state + ": " + key);
+			}
+		});
+		
+		if (bad_state) return callback(bad_state);
+		
+		this.db.connect( function(err, client, release) {
+			if (err) {
+				self.logError('pg', "Failed to acquire Postgres client for transaction commit: " + err);
+				return callback(err);
+			}
+			
+			var releaseClient = function(err) {
+				// release the pg client exactly once
+				if (!release) return;
+				var rel = release;
+				release = null;
+				rel(err);
+			};
+			
+			async.series(
+				[
+					function(callback) {
+						// start native Postgres transaction
+						query_step = 'BEGIN';
+						self.queryWithClient( client, "BEGIN", {}, function(err) {
+							if (!err) did_begin = true;
+							callback(err);
+						} );
+					},
+					function(callback) {
+						// delete all final-deleted records in one set-based SQL statement
+						if (!delete_keys.length) return callback();
+						
+						query_step = 'DELETE';
+						self.queryWithClient( client, self.commands.transDelete, {
+							keys: delete_keys
+						}, function(err, res) {
+							if (err) return callback(err);
+							
+							if (res.rowCount < delete_keys.length) {
+								self.logDebug(5, "Some records were already deleted", {
+									requested: delete_keys.length,
+									deleted: res.rowCount
+								});
+							}
+							
+							callback();
+						} );
+					},
+					function(callback) {
+						// upsert all final-written records in one set-based SQL statement
+						if (!write_keys.length) return callback();
+						
+						query_step = 'UPSERT';
+						self.queryWithClient( client, self.commands.transPut, {
+							keys: write_keys,
+							values: write_values,
+							modified: write_modified
+						}, callback );
+					},
+					function(callback) {
+						// atomically reveal all transaction actions
+						query_step = 'COMMIT';
+						self.queryWithClient( client, "COMMIT", {}, callback );
+					}
+				],
+				function(err) {
+					if (!err) {
+						// commit succeeded, so now it is safe to update the engine LRU cache
+						if (self.cache) {
+							write_keys.forEach( function(key, idx) {
+								if (!self.storage.isBinaryKey(key)) {
+									self.cache.set( key, write_values[idx], { date: write_modified[idx] } );
+								}
+							});
+							
+							delete_keys.forEach( function(key) {
+								if (!self.storage.isBinaryKey(key)) {
+									// store 'empty' stub in cache
+									self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+								}
+							});
+						}
+						
+						self.logDebug(5, "Transaction commit complete: " + trans.id, {
+							path: trans.path
+						});
+						
+						releaseClient();
+						return callback();
+					}
+					
+					self.logError('pg', "Transaction commit failed: " + err);
+					
+					// If COMMIT itself fails, the outcome may be unknowable from the app side:
+					// Postgres may have committed, but the client may have lost the response.
+					// Surface this to the outer transaction layer as a fatal consistency error.
+					if (query_step == 'COMMIT') {
+						err.fatal = true;
+					}
+					
+					if (!did_begin) {
+						releaseClient(err);
+						return callback(err);
+					}
+					
+					// rollback best-effort.  If this fails, discard the client from the pool.
+					self.queryWithClient( client, "ROLLBACK", {}, function(rollback_err) {
+						if (rollback_err) {
+							self.logError('pg', "Transaction rollback failed: " + rollback_err);
+							err.fatal = true;
+							err.rollbackError = rollback_err;
+							releaseClient(rollback_err);
+						}
+						else {
+							self.logDebug(5, "Transaction rollback complete: " + trans.id);
+							releaseClient();
+						}
+						
+						callback(err);
+					} );
+				}
+			); // series
+		} ); // db.connect
+	},
+	
+	unitTestCleanup: function(callback) {
+		// cleanup all unit test data, leaving the table itself intact
+		var self = this;
+		var table = this.config.get('table');
+		
+		this.logDebug(3, "Cleaning up Postgres unit test table: " + table);
+		
+		this.query( `TRUNCATE TABLE ${table}`, {}, function(err) {
+			if (err) {
+				self.logError('pg', "Failed to cleanup Postgres unit test table: " + err);
+				return callback(err);
+			}
+			
+			if (self.cache) self.cache.clear();
+			callback();
+		} );
+	},
+	
 	runMaintenance: function(callback) {
 		// run daily maintenance
 		callback();
 	},
 	
+	optimize: function(callback) {
+		// run Postgres VACUUM on the storage table
+		var self = this;
+		var table = this.config.get('table');
+		var perf = new Perf();
+		var vacuum = null;
+		
+		this.logDebug(3, "Running Postgres optimization", { table: table });
+		perf.setScale(1);
+		perf.begin();
+		vacuum = perf.begin('vacuum');
+		
+		this.query( `VACUUM ${table}`, {}, function(err, res) {
+			vacuum.end();
+			perf.end();
+			
+			if (err) {
+				err.message = "Postgres optimization failed: " + err.message;
+				self.logError('pg', '' + err);
+				return callback(err);
+			}
+			
+			var report = {
+				engine: self.__name,
+				optimized: true,
+				table: table,
+				perf: perf.metrics(),
+				operations: [
+					{
+						name: 'vacuum',
+						ok: true,
+						command: res.command || 'VACUUM',
+						rowCount: res.rowCount || 0
+					}
+				]
+			};
+			
+			self.logDebug(3, "Postgres optimization complete", report);
+			callback(null, report);
+		} );
+	},
+	
 	shutdown: function(callback) {
 		// shutdown storage
 		var self = this;

package/engines/Redis.js

@@ -9,6 +9,7 @@ const Class = require("pixl-class");
 const Component = require("pixl-server/component");
 const Redis = require('ioredis');
 const Tools = require("pixl-tools");
+const Perf = require("pixl-perf");
 const Cache = require("pixl-cache");
 
 module.exports = Class.create({
@@ -421,11 +422,143 @@ module.exports = Class.create({
 		} );
 	},
 	
+	commitTransaction: function(trans, callback) {
+		// commit all transaction actions using a native Redis MULTI/EXEC transaction
+		var self = this;
+		var write_keys = [];
+		var write_values = [];
+		var write_modified = [];
+		var delete_keys = [];
+		var bad_state = null;
+		var multi = this.redis.multi();
+		
+		this.logDebug(5, "Beginning transaction commit: " + trans.id, {
+			path: trans.path,
+			actions: Tools.numKeys(trans.keys)
+		});
+		
+		Object.keys(trans.keys).forEach( function(key) {
+			var record_state = trans.keys[key];
+			var norm_key = self.storage.normalizeKey(key);
+			
+			if (record_state == 'W') {
+				// Collect writes into parallel arrays so we can update the cache later
+				// without stringifying and Buffer-wrapping JSON a second time.
+				var item = trans.values[key];
+				var redis_key = self.prepKey(norm_key);
+				var value = Buffer.from( JSON.stringify( item.data ) );
+				
+				write_keys.push( redis_key );
+				write_values.push( value );
+				write_modified.push( item.mod );
+				
+				multi.set( redis_key, value );
+			}
+			else if (record_state == 'D') {
+				// Queue deletes into the same Redis transaction.
+				var redis_key = self.prepKey(norm_key);
+				delete_keys.push( redis_key );
+				
+				multi.del( redis_key );
+			}
+			else {
+				bad_state = new Error("Unknown transaction record state: " + record_state + ": " + key);
+			}
+		});
+		
+		if (bad_state) return callback(bad_state);
+		
+		multi.exec( function(err, results) {
+			if (err) {
+				// With MULTI/EXEC over the network, a top-level exec error may leave
+				// the actual Redis outcome unknowable, so treat it as fatal.
+				err.fatal = true;
+				self.logError('redis', "Transaction commit failed: " + err);
+				return callback(err);
+			}
+			
+			// Redis can return per-command errors from EXEC while still applying
+			// other commands.  That is a partial commit, so it must be fatal.
+			var cmd_err = null;
+			if (results) {
+				results.forEach( function(item) {
+					if (item && item[0] && !cmd_err) cmd_err = item[0];
+				});
+			}
+			if (cmd_err) {
+				cmd_err.fatal = true;
+				self.logError('redis', "Transaction command failed: " + cmd_err);
+				return callback(cmd_err);
+			}
+			
+			// commit succeeded, so now it is safe to update the engine LRU cache
+			if (self.cache) {
+				write_keys.forEach( function(key, idx) {
+					if (!self.storage.isBinaryKey(key)) {
+						self.cache.set( key, write_values[idx], { date: write_modified[idx] } );
+					}
+				});
+				
+				delete_keys.forEach( function(key) {
+					if (!self.storage.isBinaryKey(key)) {
+						// store 'empty' stub in cache
+						self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+					}
+				});
+			}
+			
+			self.logDebug(5, "Transaction commit complete: " + trans.id, {
+				path: trans.path
+			});
+			
+			callback();
+		} );
+	},
+	
+	unitTestCleanup: function(callback) {
+		// cleanup all unit test data
+		var self = this;
+		this.logDebug(3, "Cleaning up Redis unit test database with FLUSHDB");
+		
+		return this.redis.flushdb( function(err) {
+			if (err) {
+				self.logError('redis', "Failed to cleanup Redis unit test database: " + err);
+				return callback(err);
+			}
+			
+			if (self.cache) self.cache.clear();
+			callback();
+		} );
+	},
+	
 	runMaintenance: function(callback) {
 		// run daily maintenance
 		callback();
 	},
 	
+	optimize: function(callback) {
+		// Redis does its own memory management, so there is nothing to run here.
+		var perf = new Perf();
+		perf.setScale(1);
+		perf.begin();
+		perf.begin('noop');
+		perf.end('noop');
+		perf.end();
+		
+		process.nextTick( callback, null, {
+			engine: this.__name,
+			optimized: false,
+			perf: perf.metrics(),
+			operations: [
+				{
+					name: 'noop',
+					ok: true,
+					message: "Redis does not require storage optimization."
+				}
+			]
+		} );
+	},
+	
 	shutdown: function(callback) {
 		// shutdown storage
 		this.logDebug(2, "Shutting down Redis");

package/engines/RedisCluster.js

@@ -9,6 +9,7 @@ const Class = require("pixl-class");
 const Component = require("pixl-server/component");
 const Redis = require('ioredis');
 const Tools = require("pixl-tools");
+const Perf = require("pixl-perf");
 const Cache = require("pixl-cache");
 
 module.exports = Class.create({
@@ -439,6 +440,29 @@ module.exports = Class.create({
 		callback();
 	},
 	
+	optimize: function(callback) {
+		// Redis Cluster does its own memory management, so there is nothing to run here.
+		var perf = new Perf();
+		perf.setScale(1);
+		perf.begin();
+		perf.begin('noop');
+		perf.end('noop');
+		perf.end();
+		
+		process.nextTick( callback, null, {
+			engine: this.__name,
+			optimized: false,
+			perf: perf.metrics(),
+			operations: [
+				{
+					name: 'noop',
+					ok: true,
+					message: "RedisCluster does not require storage optimization."
+				}
+			]
+		} );
+	},
+	
 	shutdown: function(callback) {
 		// shutdown storage
 		this.logDebug(2, "Shutting down Redis");