pixl-server-storage 4.1.1 → 4.2.1

package/engines/SQLite.js

@@ -79,7 +79,7 @@ module.exports = Class.create({
 			});
 		}
 		
-		var db_file = Path.join( this.baseDir, sql_config.filename );
+		var db_file = this.dbFile = Path.join( this.baseDir, sql_config.filename );
 		this.logDebug(3, "Opening database file: " + db_file, sql_config.pragmas || {});
 		
 		try {
@@ -449,6 +449,92 @@ module.exports = Class.create({
 		});
 	},
 	
+	commitTransaction: function(trans, callback) {
+		// commit all transaction actions using a native SQLite transaction
+		var self = this;
+		var write_keys = [];
+		var write_values = [];
+		var write_modified = [];
+		var delete_keys = [];
+		var bad_state = null;
+		
+		this.logDebug(5, "Beginning transaction commit: " + trans.id, {
+			path: trans.path,
+			actions: Tools.numKeys(trans.keys)
+		});
+		
+		Object.keys(trans.keys).forEach( function(key) {
+			var record_state = trans.keys[key];
+			var norm_key = self.storage.normalizeKey(key);
+			
+			if (record_state == 'W') {
+				// Collect writes into parallel arrays so we can update the cache later
+				// without stringifying and Buffer-wrapping JSON a second time.
+				var item = trans.values[key];
+				write_keys.push( self.prepKey(norm_key) );
+				write_values.push( Buffer.from( JSON.stringify( item.data ) ) );
+				write_modified.push( item.mod );
+			}
+			else if (record_state == 'D') {
+				// Collect deletes into the same native transaction.
+				delete_keys.push( self.prepKey(norm_key) );
+			}
+			else {
+				bad_state = new Error("Unknown transaction record state: " + record_state + ": " + key);
+			}
+		});
+		
+		if (bad_state) return callback(bad_state);
+		
+		process.nextTick(function() {
+			try {
+				var put_stmt = self.db.prepare(self.commands.put);
+				var delete_stmt = self.db.prepare(self.commands.delete);
+				
+				var commit = self.db.transaction(function() {
+					delete_keys.forEach( function(key) {
+						delete_stmt.run({ key: key });
+					});
+					
+					write_keys.forEach( function(key, idx) {
+						put_stmt.run({ key: key, value: write_values[idx], now: write_modified[idx] });
+					});
+				});
+				
+				// Ask SQLite for the write transaction up front, since every commit
+				// replay here mutates rows.
+				commit.immediate();
+			}
+			catch (err) {
+				err.message = "Failed to commit transaction: " + trans.id + ": " + err;
+				self.logError('sqlite', '' + err);
+				return callback(err);
+			}
+			
+			// commit succeeded, so now it is safe to update the engine LRU cache
+			if (self.cache) {
+				write_keys.forEach( function(key, idx) {
+					if (!self.storage.isBinaryKey(key)) {
+						self.cache.set( key, write_values[idx], { date: write_modified[idx] } );
+					}
+				});
+				
+				delete_keys.forEach( function(key) {
+					if (!self.storage.isBinaryKey(key)) {
+						// store 'empty' stub in cache
+						self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+					}
+				});
+			}
+			
+			self.logDebug(5, "Transaction commit complete: " + trans.id, {
+				path: trans.path
+			});
+			
+			callback();
+		});
+	},
+	
 	runMaintenance: function(callback) {
 		// run daily maintenance
 		// config.backups: { enabled, dir, filename, compress?, keep? }
@@ -592,6 +678,112 @@ module.exports = Class.create({
 		});
 	},
 	
+	optimize: function(callback) {
+		// compact the SQLite database and run an integrity check
+		var self = this;
+		var perf = new Perf();
+		var report = {
+			engine: this.__name,
+			optimized: true,
+			file: this.dbFile,
+			operations: []
+		};
+		
+		this.logDebug(3, "Running SQLite optimization");
+		perf.setScale(1);
+		perf.begin();
+		
+		process.nextTick(function() {
+			try {
+				if (!self.db) throw new Error("SQLite database is not open.");
+				if (!self.dbFile || !fs.existsSync(self.dbFile)) throw new Error("SQLite database file could not be found.");
+				
+				var old_size = fs.statSync(self.dbFile).size;
+				
+				self.logDebug(7, "Running SQLite VACUUM");
+				perf.begin('vacuum');
+				self.db.exec("VACUUM;");
+				perf.end('vacuum');
+				
+				var new_size = fs.statSync(self.dbFile).size;
+				var bytes_reclaimed = Math.max(0, old_size - new_size);
+				
+				report.size_before = old_size;
+				report.size_after = new_size;
+				report.bytes_reclaimed = bytes_reclaimed;
+				report.operations.push({
+					name: 'vacuum',
+					ok: true,
+					size_before: old_size,
+					size_after: new_size,
+					bytes_reclaimed: bytes_reclaimed,
+					reclaimed: Tools.getTextFromBytes(bytes_reclaimed)
+				});
+				
+				// If the DB is in WAL mode, truncate the WAL file after VACUUM
+				// so the caller sees disk usage reclaimed immediately.
+				var journal_mode = '';
+				try { journal_mode = ('' + self.db.pragma('journal_mode', { simple: true })).toUpperCase(); }
+				catch (err) {
+					var pragmas = self.config.get('pragmas') || {};
+					journal_mode = ('' + (pragmas.journal_mode || '')).toUpperCase();
+				}
+				
+				if (journal_mode == 'WAL') {
+					self.logDebug(7, "Running SQLite WAL checkpoint");
+					
+					perf.begin('checkpoint');
+					var rows = self.db.prepare("PRAGMA wal_checkpoint(TRUNCATE);").all();
+					perf.end('checkpoint');
+					var row = rows && rows[0] ? rows[0] : {};
+					var busy = row.busy || 0;
+					
+					report.operations.push({
+						name: 'wal_checkpoint',
+						ok: !busy,
+						mode: 'TRUNCATE',
+						journal_mode: journal_mode,
+						busy: busy,
+						log: row.log || 0,
+						checkpointed: row.checkpointed || 0
+					});
+				}
+				
+				self.logDebug(7, "Running SQLite integrity check");
+				
+				perf.begin('integrity');
+				var rows = self.db.prepare("PRAGMA integrity_check;").all();
+				perf.end('integrity');
+				var results = rows.map( function(row) {
+					return ('' + row.integrity_check).trim();
+				}).filter( function(text) {
+					return !!text;
+				});
+				
+				var result = results.join("\n") || "No response from integrity check.";
+				var ok = (result.toLowerCase() == 'ok');
+				
+				report.integrity_ok = ok;
+				report.integrity_check = result;
+				report.operations.push({
+					name: 'integrity_check',
+					ok: ok,
+					result: result
+				});
+				
+				perf.end();
+				report.perf = perf.metrics();
+				self.logDebug(3, "SQLite optimization complete", report);
+				callback(null, report);
+			}
+			catch (err) {
+				err.message = "SQLite optimization failed: " + err.message;
+				self.logError('sqlite', '' + err);
+				callback(err);
+			}
+		});
+	},
+	
 	shutdown: function(callback) {
 		// shutdown storage
 		var self = this;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "4.1.1",
+	"version": "4.2.1",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",

package/storage.js

@@ -30,7 +30,7 @@ module.exports = Class.create({
 		maintenance: 0,
 		log_event_types: { 
 			all:0, get:0, put:0, head:0, delete:0, expire_set:0, perf_sec:0, perf_min:0,
-			commit:0, index:0, unindex:0, search:0, sort:0, maint:1 
+			commit:0, index:0, unindex:0, search:0, sort:0, maint:1, optimize:1
 		},
 		max_recent_events: 0,
 		cache_key_match: '',
@@ -787,6 +787,68 @@ module.exports = Class.create({
 		); // listEach
 	},
 	
+	optimize: function(callback) {
+		// run engine-specific storage optimization, if supported
+		var self = this;
+		if (!callback) callback = function() {};
+		if (!this.started) return callback( new Error("Storage has not completed startup.") );
+		
+		// Optimize operates on the live backing store, so it should only be run
+		// from the root storage instance, after queued writes and locks settle.
+		if (this.currentTransactionPath) {
+			return callback( new Error("Cannot optimize storage from inside a transaction.") );
+		}
+		
+		this.logDebug(3, "Starting storage optimization");
+		
+		this.waitForQueueDrain( function() {
+			self.waitForAllLocks( function() {
+				var pf = self.perf.begin('optimize');
+				var engine = self.engine;
+				var optimize = engine && engine.optimize;
+				
+				if (!optimize) {
+					pf.end();
+					var perf = new Perf();
+					perf.setScale(1);
+					perf.begin();
+					perf.begin('noop');
+					perf.end('noop');
+					perf.end();
+					
+					var report = {
+						engine: engine ? engine.__name : '',
+						optimized: false,
+						perf: perf.metrics(),
+						operations: [
+							{
+								name: 'noop',
+								ok: true,
+								message: "No optimization is available for this engine."
+							}
+						]
+					};
+					
+					self.logTransaction('optimize', report.engine || 'none', report);
+					return callback(null, report);
+				}
+				
+				optimize.call( engine, function(err, report) {
+					pf.end();
+					if (err) return callback(err);
+					
+					if (!report) report = {};
+					if (!report.engine) report.engine = engine.__name;
+					if (typeof(report.optimized) == 'undefined') report.optimized = true;
+					
+					self.logDebug(3, "Storage optimization complete", report);
+					self.logTransaction('optimize', report.engine || 'none', report);
+					callback(null, report);
+				} );
+			} ); // waitForLocks
+		} ); // waitForQueueDrain
+	},
+	
 	lock: function(key, wait, callback) {
 		// lock key in exclusive mode, possibly wait until acquired
 		if (!this.started) return callback( new Error("Storage has not completed startup.") );

package/test/config.json

@@ -14,6 +14,10 @@
 		"transactions": 0,
 		"log_event_types": { "all": 1 },
 		
+		"Hybrid": {
+			"docEngine": "SQLite",
+			"binaryEngine": "Filesystem"
+		},
 		"Filesystem": {
 			"base_dir": "data",
 			"key_namespaces": 0,

package/test/test-main.js

@@ -656,6 +656,23 @@ module.exports = {
 			} );
 		},
 		
+		function testOptimize(test) {
+			// run engine-specific optimization, or a safe no-op if unsupported
+			test.expect(8);
+			
+			this.storage.optimize( function(err, report) {
+				test.ok( !err, "No error running optimize: " + err );
+				test.ok( !!report, "Got an optimize report" );
+				test.ok( !!report.engine, "Optimize report has an engine name" );
+				test.ok( Array.isArray(report.operations), "Optimize report has operations" );
+				test.ok( report.operations.length >= 1, "Optimize report has at least one operation" );
+				test.ok( !!report.perf, "Optimize report has perf metrics" );
+				test.ok( report.perf.scale == 1, "Optimize perf metrics are in seconds" );
+				test.ok( typeof(report.perf.perf.total) == 'number', "Optimize perf metrics have a total" );
+				test.done();
+			} );
+		},
+		
 		function testMaintenance(test) {
 			var self = this;
 			test.expect(3);

package/test/test.js

@@ -67,44 +67,54 @@ module.exports = {
 				// save ref to storage
 				self.storage = server.Storage;
 				
-				// build our test array dynamically
-				// we have some repeating tests with different configuration options
-				self.tests = self.tests.concat( 
-					listTests.tests, 
+				var finishStartup = function() {
+					// build our test array dynamically
+					// we have some repeating tests with different configuration options
+					self.tests = self.tests.concat( 
+						listTests.tests, 
+						
+						(self.args.splice || self.args.all || self.args.comprehensive) ? 
+							listTests.generateSpliceTests() : [],
+						
+						hashTests.tests 
+					);
 					
-					(self.args.splice || self.args.all || self.args.comprehensive) ? 
-						listTests.generateSpliceTests() : [],
+					// now add transaction tests, which enable and init transactions
+					self.tests = self.tests.concat( transactionTests.tests );
 					
-					hashTests.tests 
-				);
-				
-				// now add transaction tests, which enable and init transactions
-				self.tests = self.tests.concat( transactionTests.tests );
-				
-				// now repeat list and hash tests, with transactions enabled
-				// augment test names for clarity
-				[].concat( listTests.tests, hashTests.tests ).forEach( function(func) {
-					var wrapper = function(test) { func.apply(this, [test]); };
-					wrapper.testName = 'Transaction_' + func.name;
-					self.tests.push( wrapper );
-				} );
-				
-				// finally add indexer tests
-				self.tests = self.tests.concat( indexerTests.tests );
-				
-				// log memory usage at end
-				self.tests.push( function(test) {
-					if (global.gc) { global.gc(); global.gc(); }
-					test.ok( true );
-					setTimeout( function() { 
-						self.storage.logDebug(9, "MEMORY USAGE: " + Tools.getTextFromBytes( process.memoryUsage.rss() ), process.memoryUsage() );
-						test.done(); 
-					}, 1 );
-				} );
+					// now repeat list and hash tests, with transactions enabled
+					// augment test names for clarity
+					[].concat( listTests.tests, hashTests.tests ).forEach( function(func) {
+						var wrapper = function(test) { func.apply(this, [test]); };
+						wrapper.testName = 'Transaction_' + func.name;
+						self.tests.push( wrapper );
+					} );
+					
+					// finally add indexer tests
+					self.tests = self.tests.concat( indexerTests.tests );
+					
+					// log memory usage at end
+					self.tests.push( function(test) {
+						if (global.gc) { global.gc(); global.gc(); }
+						test.ok( true );
+						setTimeout( function() { 
+							self.storage.logDebug(9, "MEMORY USAGE: " + Tools.getTextFromBytes( process.memoryUsage.rss() ), process.memoryUsage() );
+							test.done(); 
+						}, 1 );
+					} );
+					
+					// startup complete
+					// delay this by 1ms so the log is in the correct order (pre-start is async)
+					setTimeout( function() { callback(); }, 1 );
+				};
 				
-				// startup complete
-				// delay this by 1ms so the log is in the correct order (pre-start is async)
-				setTimeout( function() { callback(); }, 1 );
+				if (self.storage.engine.unitTestCleanup) {
+					self.storage.engine.unitTestCleanup( function(err) {
+						if (err) return callback(err);
+						finishStartup();
+					} );
+				}
+				else finishStartup();
 			} ); // startup
 		} ); // delete
 	},
@@ -127,9 +137,20 @@ module.exports = {
 	
 	tearDown: function (callback) {
 		// clean up
-		this.server.shutdown( function() {
-			cp.exec("rm -rf transactions " + base_data_dir, callback);
-		} );
+		var self = this;
+						
+		var finishShutdown = function(cleanup_err) {
+			self.server.shutdown( function(shutdown_err) {
+				cp.exec("rm -rf transactions " + base_data_dir, function(rm_err) {
+					callback( cleanup_err || shutdown_err || rm_err );
+				} );
+			} );
+		};
+		
+		if (this.storage && this.storage.engine && this.storage.engine.unitTestCleanup) {
+			this.storage.engine.unitTestCleanup( finishShutdown );
+		}
+		else finishShutdown();
 	},
 	
 	multiCheck: function(map, storage, callback) {

package/transaction.js

@@ -80,10 +80,10 @@ var TransStorageFunctions = {
 		if (trans.keys[key] == 'W') {
 			// we've written the key, so fetch our version
 			this.logDebug(9, "Pinging Object from transaction: " + key);
+			var value = trans.values[key];
 			
 			setImmediate( function() {
 				self.logDebug(9, "Head complete: " + key);
-				var value = trans.values[key];
 				callback( null, {
 					mod: value.mod,
 					len: value.len
@@ -121,11 +121,12 @@ var TransStorageFunctions = {
 		if (trans.keys[key] == 'W') {
 			// we've written the key, so fetch our version
 			this.logDebug(9, "Fetching Object in transaction: " + key);
+			var value = trans.values[key];
 			
 			setImmediate( function() {
-				var data = trans.values[key].data;
+				var data = value.data;
 				self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? data : null);
-				callback( err, Tools.copyHash(data, true) );
+				callback( null, Tools.copyHash(data, true) );
 			} );
 		}
 		else if (trans.keys[key] == 'D') {
@@ -700,6 +701,107 @@ module.exports = Class.create({
 		var num_bytes = 0;
 		var pf = this.perf.begin('commit');
 		
+		if (this.engine.commitTransaction) {
+			// native engine transaction support replaces our entire rollback-log commit system.
+			// The engine is responsible for applying all record changes atomically, using the
+			// full transaction object.  We still own the outer locks, events, logs and cleanup.
+			async.waterfall(
+				[
+					function(callback) {
+						// acquire commit lock
+						self.lock( 'C|'+path, true, function() { callback(); } );
+					},
+					function(callback) {
+						// notify listeners that the commit is starting
+						self.emit('commitStart', trans);
+						
+						// ask the engine to atomically commit all actions
+						self.engine.commitTransaction( trans, callback );
+					}
+				],
+				function(err) {
+					// commit complete
+					var elapsed = pf.end();
+					
+					if (err) {
+						var msg = "Failed to commit transaction: " + path + ": " + err.message;
+						self.logError('commit', msg, { id: trans.id });
+						
+						if (err.fatal) {
+							// Native engine detected an unsafe transaction outcome.  Do not fall
+							// through to abortTransaction(), as there is no rollback log to replay.
+							var fatal_err = new Error(msg);
+							fatal_err.fatal = true;
+							fatal_err.originalError = err;
+							return self.transFatalError(fatal_err);
+						}
+						
+						// Note: in this case unlocking happens in abortTransaction
+						return callback( new Error(msg) );
+					}
+					
+					// emulate the normal per-record put/delete side effects, since native
+					// engine commits bypass self.put() and self.delete() for atomicity.
+					Object.keys(trans.keys).forEach( function(key) {
+						var record_state = trans.keys[key];
+						var norm_key = self.normalizeKey(key);
+						
+						if (record_state == 'W') {
+							var value = trans.values[key];
+							num_bytes += value.len;
+							
+							// update the Storage-level RAM cache, if enabled for this key
+							if (self.cacheKeyRegex && norm_key.match(self.cacheKeyRegex)) {
+								self.cache[norm_key] = value.data;
+							}
+							
+							self.logTransaction('put', norm_key, {
+								elapsed_ms: 0
+							});
+							self.emit('put', norm_key, value.data);
+						}
+						else if (record_state == 'D') {
+							// update the Storage-level RAM cache, if enabled for this key
+							if (self.cacheKeyRegex && norm_key.match(self.cacheKeyRegex) && (norm_key in self.cache)) {
+								delete self.cache[norm_key];
+							}
+							
+							self.logTransaction('delete', norm_key, {
+								elapsed_ms: 0
+							});
+							self.emit('delete', norm_key);
+						}
+					});
+					
+					self.logDebug(5, "Transaction committed successfully: " + trans.id, { path: path, actions: num_actions });
+					self.logTransaction('commit', path, {
+						id: trans.id,
+						elapsed_ms: elapsed,
+						actions: num_actions,
+						bytes_written: num_bytes
+					});
+					
+					// transaction is complete
+					delete trans.values; // release memory
+					delete self.transactions[path];
+					
+					// enqueue any pending tasks that got added during the transaction
+					if (trans.queue.length) {
+						trans.queue.forEach( self.enqueue.bind(self) );
+						trans.queue = []; // release memory
+					}
+					
+					delete trans.keys; // release memory
+					
+					self.emit('commitEnd', trans);
+					self.unlock( 'C|'+path );
+					self._transUnlock(path);
+					callback();
+				}
+			); // native engine commit
+			return;
+		}
+		
 		async.waterfall(
 			[
 				function(callback) {