pixl-server-storage 4.1.0 → 4.2.0

package/engines/S3.js

@@ -149,7 +149,7 @@ module.exports = Class.create({
 		}
 		else {
 			this.logDebug(9, "Storing S3 JSON Object: " + key, this.debugLevel(10) ? params.Body : null);
-			params.Body = this.pretty ? JSON.stringify( params.Body, null, "\t" ) : JSON.stringify( params.Body );
+			params.Body = Buffer.from( this.pretty ? JSON.stringify( params.Body, null, "\t" ) : JSON.stringify( params.Body ) );
 			params.ContentType = 'application/json';
 		}
 		
@@ -257,13 +257,22 @@ module.exports = Class.create({
 		if (this.cache && this.cache.has(orig_key)) {
 			var item = this.cache.getMeta(orig_key);
 			
-			process.nextTick( function() {
-				self.logDebug(9, "Cached head complete: " + orig_key);
-				callback( null, {
-					mod: item.date,
-					len: item.value.length
+			if (item.value.length == 0) {
+				process.nextTick( function() {
+					var err = new Error("Failed to head key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+					callback( err );
 				} );
-			} );
+			}
+			else {
+				process.nextTick( function() {
+					self.logDebug(9, "Cached head complete: " + orig_key);
+					callback( null, {
+						mod: item.date,
+						len: item.value.length
+					} );
+				} );
+			}
 			return;
 		} // cache
 		
@@ -288,6 +297,11 @@ module.exports = Class.create({
 					// always include "Not found" in error message
 					err = new Error("Failed to head key: " + key + ": Not found");
 					err.code = "NoSuchKey";
+
+					if (self.cache && !self.storage.isBinaryKey(orig_key)) {
+						// store 'empty' stub in cache
+						self.cache.set( orig_key, Buffer.alloc(0), { date: 0 } );
+					}
 				}
 				else if (err.name == 'SlowDown') {
 					// special behavior for SlowDown errors
@@ -315,19 +329,29 @@ module.exports = Class.create({
 		
 		// check cache first
 		if (this.cache && !is_binary && this.cache.has(orig_key)) {
-			var data = this.cache.get(orig_key);
+			var item = this.cache.getMeta(orig_key);
 			
-			process.nextTick( function() {	
-				try { data = JSON.parse( data ); }
-				catch (e) {
-					self.logError('file', "Failed to parse JSON record: " + orig_key + ": " + e);
-					callback( e, null );
-					return;
-				}
-				self.logDebug(9, "Cached JSON fetch complete: " + orig_key, self.debugLevel(10) ? data : null);
-				
-				callback( null, data );
-			} );
+			if (item.value.length == 0) {
+				process.nextTick( function() {
+					var err = new Error("Failed to fetch key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+					callback( err );
+				} );
+			}
+			else {
+				process.nextTick( function() {
+					var data = null;
+					try { data = JSON.parse( item.value.toString() ); }
+					catch (e) {
+						self.logError('file', "Failed to parse JSON record: " + orig_key + ": " + e);
+						callback( e, null );
+						return;
+					}
+					self.logDebug(9, "Cached JSON fetch complete: " + orig_key, self.debugLevel(10) ? data : null);
+
+					callback( null, data );
+				} );
+			}
 			return;
 		} // cache
 		
@@ -350,14 +374,12 @@ module.exports = Class.create({
 						self.logDebug(9, "Binary fetch complete: " + key, '' + body.length + ' bytes');
 					}
 					else {
-						body = body.toString();
-						
 						// possibly cache in LRU
 						if (self.cache) {
 							self.cache.set( orig_key, body, { date: Tools.timeNow(true) } );
 						}
 						
-						try { body = JSON.parse( body ); }
+						try { body = JSON.parse( body.toString() ); }
 						catch (e) {
 							self.logError('s3', "Failed to parse JSON record: " + key + ": " + e);
 							callback( e, null );
@@ -378,6 +400,11 @@ module.exports = Class.create({
 					// always include "Not found" in error message
 					err = new Error("Failed to fetch key: " + key + ": Not found");
 					err.code = "NoSuchKey";
+
+					if (self.cache && !is_binary) {
+						// store 'empty' stub in cache
+						self.cache.set( orig_key, Buffer.alloc(0), { date: 0 } );
+					}
 				}
 				else if (err.name == 'SlowDown') {
 					// special behavior for SlowDown errors
@@ -577,9 +604,10 @@ module.exports = Class.create({
 				self.logDebug(9, "Delete complete: " + key);
 				self.storage.emit('billing', 's3_delete', 1);
 				
-				// possibly delete from LRU cache as well
-				if (self.cache && self.cache.has(orig_key)) {
-					self.cache.delete(orig_key);
+				// possibly "delete" from LRU cache as well
+				if (self.cache && !self.storage.isBinaryKey(orig_key)) {
+					// store 'empty' stub in cache
+					self.cache.set( orig_key, Buffer.alloc(0), { date: 0 } );
 				}
 				
 				if (callback) process.nextTick( function() { callback(null, data); } );

package/engines/SQLite.js

@@ -79,7 +79,7 @@ module.exports = Class.create({
 			});
 		}
 		
-		var db_file = Path.join( this.baseDir, sql_config.filename );
+		var db_file = this.dbFile = Path.join( this.baseDir, sql_config.filename );
 		this.logDebug(3, "Opening database file: " + db_file, sql_config.pragmas || {});
 		
 		try {
@@ -191,13 +191,22 @@ module.exports = Class.create({
 		if (this.cache && this.cache.has(key)) {
 			var item = this.cache.getMeta(key);
 			
-			process.nextTick( function() {
-				self.logDebug(9, "Cached head complete: " + key);
-				callback( null, {
-					mod: item.date,
-					len: item.value.length
+			if (item.value.length == 0) {
+				process.nextTick( function() {
+					var err = new Error("Failed to head key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+					callback( err );
 				} );
-			} );
+			}
+			else {
+				process.nextTick( function() {
+					self.logDebug(9, "Cached head complete: " + key);
+					callback( null, {
+						mod: item.date,
+						len: item.value.length
+					} );
+				} );
+			}
 			return;
 		} // cache
 		
@@ -215,6 +224,12 @@ module.exports = Class.create({
 			if (!row) {
 				var err = new Error("Failed to head key: " + key + ": Not found");
 				err.code = "NoSuchKey";
+
+				if (self.cache && !self.storage.isBinaryKey(key)) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+				}
+
 				return callback(err, null);
 			}
 			callback(null, { mod: row.modified, len: row['length(value)'] });
@@ -230,19 +245,29 @@ module.exports = Class.create({
 		
 		// check cache first
 		if (this.cache && !is_binary && this.cache.has(key)) {
-			var data = this.cache.get(key);
+			var item = this.cache.getMeta(key);
 			
-			process.nextTick( function() {
-				try { data = JSON.parse( data.toString() ); }
-				catch (e) {
-					self.logError('file', "Failed to parse JSON record: " + key + ": " + e);
-					callback( e, null );
-					return;
-				}
-				self.logDebug(9, "Cached JSON fetch complete: " + key, self.debugLevel(10) ? data : null);
-				
-				callback( null, data );
-			} );
+			if (item.value.length == 0) {
+				process.nextTick( function() {
+					var err = new Error("Failed to fetch key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+					callback( err );
+				} );
+			}
+			else {
+				process.nextTick( function() {
+					var data = null;
+					try { data = JSON.parse( item.value.toString() ); }
+					catch (e) {
+						self.logError('file', "Failed to parse JSON record: " + key + ": " + e);
+						callback( e, null );
+						return;
+					}
+					self.logDebug(9, "Cached JSON fetch complete: " + key, self.debugLevel(10) ? data : null);
+
+					callback( null, data );
+				} );
+			}
 			return;
 		} // cache
 		
@@ -263,6 +288,12 @@ module.exports = Class.create({
 			if (!row) {
 				var err = new Error("Failed to fetch key: " + key + ": Not found");
 				err.code = "NoSuchKey";
+
+				if (self.cache && !is_binary) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+				}
+
 				return callback(err, null);
 			}
 			if (is_binary) {
@@ -399,12 +430,107 @@ module.exports = Class.create({
 			if (!info.changes) {
 				var err = new Error("Failed to delete object: " + key + ": Not found");
 				err.code = "NoSuchKey";
+
+				if (self.cache && !self.storage.isBinaryKey(key)) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+				}
+
 				return callback(err);
 			}
 			self.logDebug(9, "Delete complete: " + key);
-			if (self.cache && self.cache.has(key)) {
-				self.cache.delete(key);
+
+			// possibly "delete" from LRU cache as well
+			if (self.cache && !self.storage.isBinaryKey(key)) {
+				// store 'empty' stub in cache
+				self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+			}
+			callback();
+		});
+	},
+	
+	commitTransaction: function(trans, callback) {
+		// commit all transaction actions using a native SQLite transaction
+		var self = this;
+		var write_keys = [];
+		var write_values = [];
+		var write_modified = [];
+		var delete_keys = [];
+		var bad_state = null;
+		
+		this.logDebug(5, "Beginning transaction commit: " + trans.id, {
+			path: trans.path,
+			actions: Tools.numKeys(trans.keys)
+		});
+		
+		Object.keys(trans.keys).forEach( function(key) {
+			var record_state = trans.keys[key];
+			var norm_key = self.storage.normalizeKey(key);
+			
+			if (record_state == 'W') {
+				// Collect writes into parallel arrays so we can update the cache later
+				// without stringifying and Buffer-wrapping JSON a second time.
+				var item = trans.values[key];
+				write_keys.push( self.prepKey(norm_key) );
+				write_values.push( Buffer.from( JSON.stringify( item.data ) ) );
+				write_modified.push( item.mod );
+			}
+			else if (record_state == 'D') {
+				// Collect deletes into the same native transaction.
+				delete_keys.push( self.prepKey(norm_key) );
+			}
+			else {
+				bad_state = new Error("Unknown transaction record state: " + record_state + ": " + key);
+			}
+		});
+		
+		if (bad_state) return callback(bad_state);
+		
+		process.nextTick(function() {
+			try {
+				var put_stmt = self.db.prepare(self.commands.put);
+				var delete_stmt = self.db.prepare(self.commands.delete);
+				
+				var commit = self.db.transaction(function() {
+					delete_keys.forEach( function(key) {
+						delete_stmt.run({ key: key });
+					});
+					
+					write_keys.forEach( function(key, idx) {
+						put_stmt.run({ key: key, value: write_values[idx], now: write_modified[idx] });
+					});
+				});
+				
+				// Ask SQLite for the write transaction up front, since every commit
+				// replay here mutates rows.
+				commit.immediate();
+			}
+			catch (err) {
+				err.message = "Failed to commit transaction: " + trans.id + ": " + err;
+				self.logError('sqlite', '' + err);
+				return callback(err);
 			}
+			
+			// commit succeeded, so now it is safe to update the engine LRU cache
+			if (self.cache) {
+				write_keys.forEach( function(key, idx) {
+					if (!self.storage.isBinaryKey(key)) {
+						self.cache.set( key, write_values[idx], { date: write_modified[idx] } );
+					}
+				});
+				
+				delete_keys.forEach( function(key) {
+					if (!self.storage.isBinaryKey(key)) {
+						// store 'empty' stub in cache
+						self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+					}
+				});
+			}
+			
+			self.logDebug(5, "Transaction commit complete: " + trans.id, {
+				path: trans.path
+			});
+			
 			callback();
 		});
 	},
@@ -552,6 +678,112 @@ module.exports = Class.create({
 		});
 	},
 	
+	optimize: function(callback) {
+		// compact the SQLite database and run an integrity check
+		var self = this;
+		var perf = new Perf();
+		var report = {
+			engine: this.__name,
+			optimized: true,
+			file: this.dbFile,
+			operations: []
+		};
+		
+		this.logDebug(3, "Running SQLite optimization");
+		perf.setScale(1);
+		perf.begin();
+		
+		process.nextTick(function() {
+			try {
+				if (!self.db) throw new Error("SQLite database is not open.");
+				if (!self.dbFile || !fs.existsSync(self.dbFile)) throw new Error("SQLite database file could not be found.");
+				
+				var old_size = fs.statSync(self.dbFile).size;
+				
+				self.logDebug(7, "Running SQLite VACUUM");
+				perf.begin('vacuum');
+				self.db.exec("VACUUM;");
+				perf.end('vacuum');
+				
+				var new_size = fs.statSync(self.dbFile).size;
+				var bytes_reclaimed = Math.max(0, old_size - new_size);
+				
+				report.size_before = old_size;
+				report.size_after = new_size;
+				report.bytes_reclaimed = bytes_reclaimed;
+				report.operations.push({
+					name: 'vacuum',
+					ok: true,
+					size_before: old_size,
+					size_after: new_size,
+					bytes_reclaimed: bytes_reclaimed,
+					reclaimed: Tools.getTextFromBytes(bytes_reclaimed)
+				});
+				
+				// If the DB is in WAL mode, truncate the WAL file after VACUUM
+				// so the caller sees disk usage reclaimed immediately.
+				var journal_mode = '';
+				try { journal_mode = ('' + self.db.pragma('journal_mode', { simple: true })).toUpperCase(); }
+				catch (err) {
+					var pragmas = self.config.get('pragmas') || {};
+					journal_mode = ('' + (pragmas.journal_mode || '')).toUpperCase();
+				}
+				
+				if (journal_mode == 'WAL') {
+					self.logDebug(7, "Running SQLite WAL checkpoint");
+					
+					perf.begin('checkpoint');
+					var rows = self.db.prepare("PRAGMA wal_checkpoint(TRUNCATE);").all();
+					perf.end('checkpoint');
+					var row = rows && rows[0] ? rows[0] : {};
+					var busy = row.busy || 0;
+					
+					report.operations.push({
+						name: 'wal_checkpoint',
+						ok: !busy,
+						mode: 'TRUNCATE',
+						journal_mode: journal_mode,
+						busy: busy,
+						log: row.log || 0,
+						checkpointed: row.checkpointed || 0
+					});
+				}
+				
+				self.logDebug(7, "Running SQLite integrity check");
+				
+				perf.begin('integrity');
+				var rows = self.db.prepare("PRAGMA integrity_check;").all();
+				perf.end('integrity');
+				var results = rows.map( function(row) {
+					return ('' + row.integrity_check).trim();
+				}).filter( function(text) {
+					return !!text;
+				});
+				
+				var result = results.join("\n") || "No response from integrity check.";
+				var ok = (result.toLowerCase() == 'ok');
+				
+				report.integrity_ok = ok;
+				report.integrity_check = result;
+				report.operations.push({
+					name: 'integrity_check',
+					ok: ok,
+					result: result
+				});
+				
+				perf.end();
+				report.perf = perf.metrics();
+				self.logDebug(3, "SQLite optimization complete", report);
+				callback(null, report);
+			}
+			catch (err) {
+				err.message = "SQLite optimization failed: " + err.message;
+				self.logError('sqlite', '' + err);
+				callback(err);
+			}
+		});
+	},
+	
 	shutdown: function(callback) {
 		// shutdown storage
 		var self = this;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "4.1.0",
+	"version": "4.2.0",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",

package/storage.js

@@ -30,7 +30,7 @@ module.exports = Class.create({
 		maintenance: 0,
 		log_event_types: { 
 			all:0, get:0, put:0, head:0, delete:0, expire_set:0, perf_sec:0, perf_min:0,
-			commit:0, index:0, unindex:0, search:0, sort:0, maint:1 
+			commit:0, index:0, unindex:0, search:0, sort:0, maint:1, optimize:1
 		},
 		max_recent_events: 0,
 		cache_key_match: '',
@@ -787,6 +787,68 @@ module.exports = Class.create({
 		); // listEach
 	},
 	
+	optimize: function(callback) {
+		// run engine-specific storage optimization, if supported
+		var self = this;
+		if (!callback) callback = function() {};
+		if (!this.started) return callback( new Error("Storage has not completed startup.") );
+		
+		// Optimize operates on the live backing store, so it should only be run
+		// from the root storage instance, after queued writes and locks settle.
+		if (this.currentTransactionPath) {
+			return callback( new Error("Cannot optimize storage from inside a transaction.") );
+		}
+		
+		this.logDebug(3, "Starting storage optimization");
+		
+		this.waitForQueueDrain( function() {
+			self.waitForAllLocks( function() {
+				var pf = self.perf.begin('optimize');
+				var engine = self.engine;
+				var optimize = engine && engine.optimize;
+				
+				if (!optimize) {
+					pf.end();
+					var perf = new Perf();
+					perf.setScale(1);
+					perf.begin();
+					perf.begin('noop');
+					perf.end('noop');
+					perf.end();
+					
+					var report = {
+						engine: engine ? engine.__name : '',
+						optimized: false,
+						perf: perf.metrics(),
+						operations: [
+							{
+								name: 'noop',
+								ok: true,
+								message: "No optimization is available for this engine."
+							}
+						]
+					};
+					
+					self.logTransaction('optimize', report.engine || 'none', report);
+					return callback(null, report);
+				}
+				
+				optimize.call( engine, function(err, report) {
+					pf.end();
+					if (err) return callback(err);
+					
+					if (!report) report = {};
+					if (!report.engine) report.engine = engine.__name;
+					if (typeof(report.optimized) == 'undefined') report.optimized = true;
+					
+					self.logDebug(3, "Storage optimization complete", report);
+					self.logTransaction('optimize', report.engine || 'none', report);
+					callback(null, report);
+				} );
+			} ); // waitForLocks
+		} ); // waitForQueueDrain
+	},
+	
 	lock: function(key, wait, callback) {
 		// lock key in exclusive mode, possibly wait until acquired
 		if (!this.started) return callback( new Error("Storage has not completed startup.") );

package/test/config.json

@@ -14,6 +14,10 @@
 		"transactions": 0,
 		"log_event_types": { "all": 1 },
 		
+		"Hybrid": {
+			"docEngine": "SQLite",
+			"binaryEngine": "Filesystem"
+		},
 		"Filesystem": {
 			"base_dir": "data",
 			"key_namespaces": 0,

package/test/test-main.js

@@ -656,6 +656,23 @@ module.exports = {
 			} );
 		},
 		
+		function testOptimize(test) {
+			// run engine-specific optimization, or a safe no-op if unsupported
+			test.expect(8);
+			
+			this.storage.optimize( function(err, report) {
+				test.ok( !err, "No error running optimize: " + err );
+				test.ok( !!report, "Got an optimize report" );
+				test.ok( !!report.engine, "Optimize report has an engine name" );
+				test.ok( Array.isArray(report.operations), "Optimize report has operations" );
+				test.ok( report.operations.length >= 1, "Optimize report has at least one operation" );
+				test.ok( !!report.perf, "Optimize report has perf metrics" );
+				test.ok( report.perf.scale == 1, "Optimize perf metrics are in seconds" );
+				test.ok( typeof(report.perf.perf.total) == 'number', "Optimize perf metrics have a total" );
+				test.done();
+			} );
+		},
+		
 		function testMaintenance(test) {
 			var self = this;
 			test.expect(3);