pixl-server-storage 4.0.0 → 4.0.2

package/engines/SQLite.js

@@ -148,18 +148,19 @@ module.exports = Class.create({
 		process.nextTick(function() {
 			try {
 				self.db.prepare(self.commands.put).run({ key: key, value: value, now: now });
-				self.logDebug(9, "Store complete: " + key);
-				// possibly cache in LRU
-				if (self.cache && !is_binary) {
-					self.cache.set( key, value, { date: Tools.timeNow(true) } );
-				}
-				if (callback) callback(null);
 			}
 			catch (err) {
 				err.message = "Failed to store object: " + key + ": " + err;
 				self.logError('sqlite', '' + err);
 				if (callback) callback(err);
 			}
+			
+			self.logDebug(9, "Store complete: " + key);
+			// possibly cache in LRU
+			if (self.cache && !is_binary) {
+				self.cache.set( key, value, { date: Tools.timeNow(true) } );
+			}
+			if (callback) callback(null);
 		});
 	},
 	
@@ -200,20 +201,22 @@ module.exports = Class.create({
 		} // cache
 		
 		process.nextTick(function() {
+			var row = null;
 			try {
-				var row = self.db.prepare(self.commands.head).get({ key: key });
-				if (!row) {
-					var err = new Error("Failed to head key: " + key + ": Not found");
-					err.code = "NoSuchKey";
-					return callback(err, null);
-				}
-				callback(null, { mod: row.modified, len: row['length(value)'] });
+				row = self.db.prepare(self.commands.head).get({ key: key });
 			}
 			catch (err) {
 				err.message = "Failed to head key: " + key + ": " + err;
 				self.logError('sqlite', '' + err);
 				callback(err);
 			}
+			
+			if (!row) {
+				var err = new Error("Failed to head key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback(err, null);
+			}
+			callback(null, { mod: row.modified, len: row['length(value)'] });
 		});
 	},
 	
@@ -245,34 +248,36 @@ module.exports = Class.create({
 		this.logDebug(9, "Fetching SQLite Object: " + key);
 		
 		process.nextTick(function() {
+			var row = null;
 			try {
-				var row = self.db.prepare(self.commands.get).get({ key: key });
-				if (!row) {
-					var err = new Error("Failed to fetch key: " + key + ": Not found");
-					err.code = "NoSuchKey";
-					return callback(err, null);
-				}
-				if (is_binary) {
-					self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
-					return callback(null, row.value);
-				}
-				if (self.cache) {
-					self.cache.set( key, row.value, { date: Tools.timeNow(true) } );
-				}
-				var json = null;
-				try { json = JSON.parse( row.value.toString() ); }
-				catch (err) {
-					self.logError('sqlite', "Failed to parse JSON record: " + key + ": " + err);
-					return callback(err, null);
-				}
-				self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? json : null);
-				callback(null, json);
+				row = self.db.prepare(self.commands.get).get({ key: key });
 			}
 			catch (err) {
 				err.message = "Failed to fetch key: " + key + ": " + err;
 				self.logError('sqlite', '' + err);
 				callback(err);
 			}
+			
+			if (!row) {
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback(err, null);
+			}
+			if (is_binary) {
+				self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
+				return callback(null, row.value);
+			}
+			if (self.cache) {
+				self.cache.set( key, row.value, { date: Tools.timeNow(true) } );
+			}
+			var json = null;
+			try { json = JSON.parse( row.value.toString() ); }
+			catch (err) {
+				self.logError('sqlite', "Failed to parse JSON record: " + key + ": " + err);
+				return callback(err, null);
+			}
+			self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? json : null);
+			callback(null, json);
 		});
 	},
 	
@@ -284,21 +289,23 @@ module.exports = Class.create({
 		this.logDebug(9, "Fetching SQLite Object: " + key);
 		
 		process.nextTick(function() {
+			var row = null;
 			try {
-				var row = self.db.prepare(self.commands.get).get({ key: key });
-				if (!row) {
-					var err = new Error("Failed to fetch key: " + key + ": Not found");
-					err.code = "NoSuchKey";
-					return callback(err, null);
-				}
-				self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
-				callback(null, row.value);
+				row = self.db.prepare(self.commands.get).get({ key: key });
 			}
 			catch (err) {
 				err.message = "Failed to fetch key: " + key + ": " + err;
 				self.logError('sqlite', '' + err);
 				callback(err);
 			}
+			
+			if (!row) {
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback(err, null);
+			}
+			self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
+			callback(null, row.value);
 		});
 	},
 	
@@ -377,23 +384,25 @@ module.exports = Class.create({
 		this.logDebug(9, "Deleting SQLite Object: " + key);
 		
 		process.nextTick(function() {
+			var info = null;
 			try {
-				var info = self.db.prepare(self.commands.delete).run({ key: key });
-				if (!info.changes) {
-					var err = new Error("Failed to delete object: " + key + ": Not found");
-					err.code = "NoSuchKey";
-					return callback(err);
-				}
-				self.logDebug(9, "Delete complete: " + key);
-				if (self.cache && self.cache.has(key)) {
-					self.cache.delete(key);
-				}
-				callback();
+				info = self.db.prepare(self.commands.delete).run({ key: key });
 			}
 			catch (err) {
 				self.logError('sqlite', "Failed to delete object: " + key + ": " + err);
 				callback(err);
 			}
+			
+			if (!info.changes) {
+				var err = new Error("Failed to delete object: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback(err);
+			}
+			self.logDebug(9, "Delete complete: " + key);
+			if (self.cache && self.cache.has(key)) {
+				self.cache.delete(key);
+			}
+			callback();
 		});
 	},
 	

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "4.0.0",
+	"version": "4.0.2",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -22,9 +22,9 @@
 		"sqlite"
 	],
 	"dependencies": {
-		"@aws-sdk/client-s3": "3.621.0",
-		"@aws-sdk/lib-storage": "3.621.0",
-		"@smithy/node-http-handler": "3.1.4",
+		"@aws-sdk/client-s3": "3.995.0",
+		"@aws-sdk/lib-storage": "3.995.0",
+		"@smithy/node-http-handler": "3.3.3",
 		"async": "2.6.4",
 		"fast-stream-to-buffer": "1.0.0",
 		"he": "1.1.1",

package/test/config.json

@@ -76,6 +76,13 @@
 				"enabled": true,
 				"maxItems": 1000,
 				"maxBytes": 10485760
+			},
+			"backups": {
+				"enabled": true,
+				"dir": "data/backups",
+				"filename": "backup-[yyyy]-[mm]-[dd]-[hh]-[mi]-[ss].db",
+				"compress": true,
+				"keep": 7
 			}
 		}
 	}

package/test/test-indexer.js

@@ -9,8 +9,11 @@ var cp = require('child_process');
 var async = require('async');
 var Tools = require('pixl-tools');
 
+var sample_tickets = null;
+var fixtures = null;
+
 var sample_data = require('./sample-data.json');
-var sample_tickets = sample_data.Ticket;
+var orig_sample_tickets = sample_data.Ticket;
 
 var index_config = {
 	base_path: "/index/ontrack",
@@ -50,7 +53,7 @@ var index_config = {
 	remove_words: ["the","of","and","a","to","in","is","you","that","it","he","was","for","on","are","as","with","his","they","I","at","be","this","have","from","or","one","had","by","word","but","not","what","all","were","we","when","your","can","said","there","use","an","each","which","she","do","how","their","if","will","up","other","about","out","many","then","them","these","so","some","her","would","make","like","him","into","time","has","look","two","more","write","go","see","number","no","way","could","people","my","than","first","water","been","call","who","oil","its","now","find","long","down","day","did","get","come","made","may","part"]
 };
 
-var fixtures = {
+var orig_fixtures = {
 	searchRecordsExact2: {
 		'title:"Released to Preproduction"': { "2653": 1, "2654": 1, "2659": 1, "2662": 1, "2665": 1 },
 		'status:open title:"Released to Preproduction"': { "2653": 1, "2654": 1 },
@@ -317,6 +320,13 @@ var fixtures = {
 
 module.exports = {
 	tests: [
+		
+		function resetFixtures(test) {
+			fixtures = Tools.copyHash( orig_fixtures, true );
+			sample_tickets = Tools.copyHash( orig_sample_tickets, true );
+			test.ok( true );
+			test.done();
+		},
 	
 		// insert record
 		

package/test/test.js

@@ -51,6 +51,7 @@ module.exports = {
 			
 			// override Filesystem base dir to go somewhere more sane
 			if (storage_config.Filesystem) storage_config.Filesystem.base_dir = base_data_dir;
+			if (storage_config.SQLite) storage_config.SQLite.base_dir = base_data_dir;
 		});
 		
 		// delete old unit test log
@@ -58,6 +59,7 @@ module.exports = {
 			// startup mock server
 			server.startup( function() {
 				// startup complete
+				server.logger.debug(9, "BASE DIR: " + base_data_dir);
 				
 				// write log in sync mode, for troubleshooting
 				server.logger.set('sync', true);
@@ -90,6 +92,16 @@ module.exports = {
 				// finally add indexer tests
 				self.tests = self.tests.concat( indexerTests.tests );
 				
+				// log memory usage at end
+				self.tests.push( function(test) {
+					if (global.gc) { global.gc(); global.gc(); }
+					test.ok( true );
+					setTimeout( function() { 
+						self.storage.logDebug(9, "MEMORY USAGE: " + Tools.getTextFromBytes( process.memoryUsage.rss() ), process.memoryUsage() );
+						test.done(); 
+					}, 1 );
+				} );
+				
 				// startup complete
 				// delay this by 1ms so the log is in the correct order (pre-start is async)
 				setTimeout( function() { callback(); }, 1 );