pixl-server-storage 4.1.0 → 4.2.0

package/test/test.js

@@ -67,44 +67,54 @@ module.exports = {
 				// save ref to storage
 				self.storage = server.Storage;
 				
-				// build our test array dynamically
-				// we have some repeating tests with different configuration options
-				self.tests = self.tests.concat( 
-					listTests.tests, 
+				var finishStartup = function() {
+					// build our test array dynamically
+					// we have some repeating tests with different configuration options
+					self.tests = self.tests.concat( 
+						listTests.tests, 
+						
+						(self.args.splice || self.args.all || self.args.comprehensive) ? 
+							listTests.generateSpliceTests() : [],
+						
+						hashTests.tests 
+					);
 					
-					(self.args.splice || self.args.all || self.args.comprehensive) ? 
-						listTests.generateSpliceTests() : [],
+					// now add transaction tests, which enable and init transactions
+					self.tests = self.tests.concat( transactionTests.tests );
 					
-					hashTests.tests 
-				);
-				
-				// now add transaction tests, which enable and init transactions
-				self.tests = self.tests.concat( transactionTests.tests );
-				
-				// now repeat list and hash tests, with transactions enabled
-				// augment test names for clarity
-				[].concat( listTests.tests, hashTests.tests ).forEach( function(func) {
-					var wrapper = function(test) { func.apply(this, [test]); };
-					wrapper.testName = 'Transaction_' + func.name;
-					self.tests.push( wrapper );
-				} );
-				
-				// finally add indexer tests
-				self.tests = self.tests.concat( indexerTests.tests );
-				
-				// log memory usage at end
-				self.tests.push( function(test) {
-					if (global.gc) { global.gc(); global.gc(); }
-					test.ok( true );
-					setTimeout( function() { 
-						self.storage.logDebug(9, "MEMORY USAGE: " + Tools.getTextFromBytes( process.memoryUsage.rss() ), process.memoryUsage() );
-						test.done(); 
-					}, 1 );
-				} );
+					// now repeat list and hash tests, with transactions enabled
+					// augment test names for clarity
+					[].concat( listTests.tests, hashTests.tests ).forEach( function(func) {
+						var wrapper = function(test) { func.apply(this, [test]); };
+						wrapper.testName = 'Transaction_' + func.name;
+						self.tests.push( wrapper );
+					} );
+					
+					// finally add indexer tests
+					self.tests = self.tests.concat( indexerTests.tests );
+					
+					// log memory usage at end
+					self.tests.push( function(test) {
+						if (global.gc) { global.gc(); global.gc(); }
+						test.ok( true );
+						setTimeout( function() { 
+							self.storage.logDebug(9, "MEMORY USAGE: " + Tools.getTextFromBytes( process.memoryUsage.rss() ), process.memoryUsage() );
+							test.done(); 
+						}, 1 );
+					} );
+					
+					// startup complete
+					// delay this by 1ms so the log is in the correct order (pre-start is async)
+					setTimeout( function() { callback(); }, 1 );
+				};
 				
-				// startup complete
-				// delay this by 1ms so the log is in the correct order (pre-start is async)
-				setTimeout( function() { callback(); }, 1 );
+				if (self.storage.engine.unitTestCleanup) {
+					self.storage.engine.unitTestCleanup( function(err) {
+						if (err) return callback(err);
+						finishStartup();
+					} );
+				}
+				else finishStartup();
 			} ); // startup
 		} ); // delete
 	},
@@ -127,9 +137,20 @@ module.exports = {
 	
 	tearDown: function (callback) {
 		// clean up
-		this.server.shutdown( function() {
-			cp.exec("rm -rf transactions " + base_data_dir, callback);
-		} );
+		var self = this;
+						
+		var finishShutdown = function(cleanup_err) {
+			self.server.shutdown( function(shutdown_err) {
+				cp.exec("rm -rf transactions " + base_data_dir, function(rm_err) {
+					callback( cleanup_err || shutdown_err || rm_err );
+				} );
+			} );
+		};
+		
+		if (this.storage && this.storage.engine && this.storage.engine.unitTestCleanup) {
+			this.storage.engine.unitTestCleanup( finishShutdown );
+		}
+		else finishShutdown();
 	},
 	
 	multiCheck: function(map, storage, callback) {

package/transaction.js

@@ -80,10 +80,10 @@ var TransStorageFunctions = {
 		if (trans.keys[key] == 'W') {
 			// we've written the key, so fetch our version
 			this.logDebug(9, "Pinging Object from transaction: " + key);
+			var value = trans.values[key];
 			
 			setImmediate( function() {
 				self.logDebug(9, "Head complete: " + key);
-				var value = trans.values[key];
 				callback( null, {
 					mod: value.mod,
 					len: value.len
@@ -121,11 +121,12 @@ var TransStorageFunctions = {
 		if (trans.keys[key] == 'W') {
 			// we've written the key, so fetch our version
 			this.logDebug(9, "Fetching Object in transaction: " + key);
+			var value = trans.values[key];
 			
 			setImmediate( function() {
-				var data = trans.values[key].data;
+				var data = value.data;
 				self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? data : null);
-				callback( err, Tools.copyHash(data, true) );
+				callback( null, Tools.copyHash(data, true) );
 			} );
 		}
 		else if (trans.keys[key] == 'D') {
@@ -700,6 +701,107 @@ module.exports = Class.create({
 		var num_bytes = 0;
 		var pf = this.perf.begin('commit');
 		
+		if (this.engine.commitTransaction) {
+			// native engine transaction support replaces our entire rollback-log commit system.
+			// The engine is responsible for applying all record changes atomically, using the
+			// full transaction object.  We still own the outer locks, events, logs and cleanup.
+			async.waterfall(
+				[
+					function(callback) {
+						// acquire commit lock
+						self.lock( 'C|'+path, true, function() { callback(); } );
+					},
+					function(callback) {
+						// notify listeners that the commit is starting
+						self.emit('commitStart', trans);
+						
+						// ask the engine to atomically commit all actions
+						self.engine.commitTransaction( trans, callback );
+					}
+				],
+				function(err) {
+					// commit complete
+					var elapsed = pf.end();
+					
+					if (err) {
+						var msg = "Failed to commit transaction: " + path + ": " + err.message;
+						self.logError('commit', msg, { id: trans.id });
+						
+						if (err.fatal) {
+							// Native engine detected an unsafe transaction outcome.  Do not fall
+							// through to abortTransaction(), as there is no rollback log to replay.
+							var fatal_err = new Error(msg);
+							fatal_err.fatal = true;
+							fatal_err.originalError = err;
+							return self.transFatalError(fatal_err);
+						}
+						
+						// Note: in this case unlocking happens in abortTransaction
+						return callback( new Error(msg) );
+					}
+					
+					// emulate the normal per-record put/delete side effects, since native
+					// engine commits bypass self.put() and self.delete() for atomicity.
+					Object.keys(trans.keys).forEach( function(key) {
+						var record_state = trans.keys[key];
+						var norm_key = self.normalizeKey(key);
+						
+						if (record_state == 'W') {
+							var value = trans.values[key];
+							num_bytes += value.len;
+							
+							// update the Storage-level RAM cache, if enabled for this key
+							if (self.cacheKeyRegex && norm_key.match(self.cacheKeyRegex)) {
+								self.cache[norm_key] = value.data;
+							}
+							
+							self.logTransaction('put', norm_key, {
+								elapsed_ms: 0
+							});
+							self.emit('put', norm_key, value.data);
+						}
+						else if (record_state == 'D') {
+							// update the Storage-level RAM cache, if enabled for this key
+							if (self.cacheKeyRegex && norm_key.match(self.cacheKeyRegex) && (norm_key in self.cache)) {
+								delete self.cache[norm_key];
+							}
+							
+							self.logTransaction('delete', norm_key, {
+								elapsed_ms: 0
+							});
+							self.emit('delete', norm_key);
+						}
+					});
+					
+					self.logDebug(5, "Transaction committed successfully: " + trans.id, { path: path, actions: num_actions });
+					self.logTransaction('commit', path, {
+						id: trans.id,
+						elapsed_ms: elapsed,
+						actions: num_actions,
+						bytes_written: num_bytes
+					});
+					
+					// transaction is complete
+					delete trans.values; // release memory
+					delete self.transactions[path];
+					
+					// enqueue any pending tasks that got added during the transaction
+					if (trans.queue.length) {
+						trans.queue.forEach( self.enqueue.bind(self) );
+						trans.queue = []; // release memory
+					}
+					
+					delete trans.keys; // release memory
+					
+					self.emit('commitEnd', trans);
+					self.unlock( 'C|'+path );
+					self._transUnlock(path);
+					callback();
+				}
+			); // native engine commit
+			return;
+		}
+		
 		async.waterfall(
 			[
 				function(callback) {