pixl-server-storage 3.1.0 → 3.1.2

package/README.md

@@ -611,6 +611,32 @@ The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key
 
 The optional `keyTemplate` property works similarly to the [S3 Key Template](#s3-key-template) feature.  It allows you to specify an exact layout of MD5 hash characters, which can be prefixed, mixed in with or postfixed after the key.
 
+## SQLite
+
+If you want to use [SQLite](https://sqlite.com/) as a backing store, here is how to do so.  First, you need to manually install the [sqlite3](https://www.npmjs.com/package/sqlite3) module into your app:
+
+```
+npm install --save sqlite3
+```
+
+Then configure your storage thusly:
+
+```javascript
+{
+	"engine": "SQLite",
+	"SQLite": {
+		"base_dir": "data",
+		"filename": "sqlite.db"
+	}
+}
+```
+
+The `base_dir` defaults to the current working directory, and will be created on startup if necessary.
+
+The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key-prefix) feature.  It allows you to prefix all the SQLite keys with a common string, to separate your application's data in a shared database situation.
+
+The optional `keyTemplate` property works similarly to the [S3 Key Template](#s3-key-template) feature.  It allows you to specify an exact layout of MD5 hash characters, which can be prefixed, mixed in with or postfixed after the key.
+
 ## Hybrid
 
 Your application may need the features of multiple engines.  Specifically, you may want JSON (document) records to use one engine, and binary records to use another.  Binary records are specified with keys that end in a file extension, e.g. `.jpg`.  To facilitate this, there is a `Hybrid` engine available, which can load multiple sub-engines, one for JSON keys and one for binary keys.  Example use:

package/engines/SQLite.js

@@ -0,0 +1,363 @@
+// SQLite Storage Plugin
+// Copyright (c) 2023 Joseph Huckaby
+// Released under the MIT License
+
+// Requires the 'sqlite3' module from npm
+// npm install --save sqlite3
+
+const Path = require('path');
+const Class = require("pixl-class");
+const Component = require("pixl-server/component");
+const SQLite3 = require('sqlite3');
+const Tools = require("pixl-tools");
+const async = require('async');
+
+module.exports = Class.create({
+	
+	__name: 'SQLite',
+	__parent: Component,
+	
+	defaultConfig: {
+		base_dir: '',
+		filename: 'sqlite.db',
+		keyPrefix: "",
+		keyTemplate: ""
+	},
+	
+	startup: function(callback) {
+		// setup SQLite connection
+		var self = this;
+		this.logDebug(2, "Setting up SQLite", this.config.get() );
+		this.setup(callback);
+	},
+	
+	setup: function(callback) {
+		// setup SQLite connection
+		var self = this;
+		var sql_config = this.config.get();
+		
+		this.keyPrefix = (sql_config.keyPrefix || '').replace(/^\//, '');
+		if (this.keyPrefix && !this.keyPrefix.match(/\/$/)) this.keyPrefix += '/';
+		
+		this.keyTemplate = (sql_config.keyTemplate || '').replace(/^\//, '').replace(/\/$/, '');
+		
+		this.baseDir = sql_config.base_dir || process.cwd();
+		this.commands = {};
+		
+		// create initial data dir if necessary
+		try {
+			Tools.mkdirp.sync( this.baseDir ); 
+		}
+		catch (e) {
+			var msg = "FATAL ERROR: Base directory could not be created: " + this.baseDir + ": " + e;
+			this.logError('sqlite', msg);
+			throw new Error(msg);
+		}
+		
+		var db_file = Path.join( this.baseDir, sql_config.filename );
+		this.logDebug(3, "Opening database file: " + db_file);
+		
+		async.series([
+			function(callback) {
+				self.db = new SQLite3.Database( db_file, callback );
+			},
+			function(callback) {
+				self.db.run( 'CREATE TABLE IF NOT EXISTS items( key TEXT PRIMARY KEY, value BLOB, modified INTEGER )', callback );
+			},
+			function(callback) {
+				self.commands.get = self.db.prepare( 'SELECT value FROM items WHERE key = $key LIMIT 1', callback );
+			},
+			function(callback) {
+				self.commands.head = self.db.prepare( 'SELECT modified, length(value) FROM items WHERE key = $key LIMIT 1', callback );
+			},
+			function(callback) {
+				self.commands.put = self.db.prepare( 'INSERT INTO items VALUES($key, $value, $now) ON CONFLICT (key) DO UPDATE SET value = $value, modified = $now WHERE key = $key', callback );
+			},
+			function(callback) {
+				self.commands.delete = self.db.prepare( 'DELETE FROM items WHERE key = $key', callback );
+			}
+		], 
+		function(err) {
+			if (err) {
+				self.logError('sqlite', "FATAL ERROR: Database setup failed: " + err);
+				return callback(err);
+			}
+			self.logDebug(3, "Setup complete");
+			callback();
+		}); // async.series
+	},
+	
+	prepKey: function(key) {
+		// prepare key based on config
+		var md5 = Tools.digestHex(key, 'md5');
+		
+		if (this.keyPrefix) {
+			key = this.keyPrefix + key;
+		}
+		
+		if (this.keyTemplate) {
+			var idx = 0;
+			var temp = this.keyTemplate.replace( /\#/g, function() {
+				return md5.substr(idx++, 1);
+			} );
+			key = Tools.substitute( temp, { key: key, md5: md5 } );
+		}
+		
+		return key;
+	},
+	
+	put: function(key, value, callback) {
+		// store key+value in SQLite
+		var self = this;
+		var now = Tools.timeNow(true);
+		key = this.prepKey(key);
+		
+		if (this.storage.isBinaryKey(key)) {
+			this.logDebug(9, "Storing SQLite Binary Object: " + key, '' + value.length + ' bytes');
+		}
+		else {
+			this.logDebug(9, "Storing SQLite JSON Object: " + key, this.debugLevel(10) ? value : null);
+			value = Buffer.from( JSON.stringify( value ) );
+		}
+		
+		this.commands.put.run({ $key: key, $value: value, $now: now }, function(err) {
+			if (err) {
+				err.message = "Failed to store object: " + key + ": " + err;
+				self.logError('sqlite', '' + err);
+			}
+			else self.logDebug(9, "Store complete: " + key);
+			
+			if (callback) callback(err);
+		} ); // put
+	},
+	
+	putStream: function(key, inp, callback) {
+		// store key+value in SQLite using read stream
+		var self = this;
+		
+		// The SQLite API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		
+		var chunks = [];
+		inp.on('data', function(chunk) {
+			chunks.push( chunk );
+		} );
+		inp.on('end', function() {
+			var buf = Buffer.concat(chunks);
+			self.put( key, buf, callback );
+		} );
+	},
+	
+	head: function(key, callback) {
+		// head sqlite item given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		this.commands.head.get({ $key: key }, function(err, row) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to head key: " + key + ": " + err;
+				self.logError('sqlite', '' + err);
+				callback(err);
+			}
+			else if (!row) {
+				// record not found
+				// always use "NoSuchKey" in error code
+				var err = new Error("Failed to head key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				callback( err, null );
+			}
+			else {
+				callback( null, { mod: row.modified, len: row['length(value)'] } );
+			}
+		}); // head
+	},
+	
+	get: function(key, callback) {
+		// fetch SQLite value given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Fetching SQLite Object: " + key);
+		
+		this.commands.get.get({ $key: key }, function(err, row) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('sqlite', '' + err);
+				callback(err);
+			}
+			else if (!row) {
+				// record not found
+				// always use "NoSuchKey" in error code
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				callback( err, null );
+			}
+			else {
+				// success
+				if (self.storage.isBinaryKey(key)) {
+					self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
+					callback( null, row.value );
+				}
+				else {
+					var json = null;
+					try { json = JSON.parse( row.value.toString() ); }
+					catch (err) {
+						self.logError('sqlite', "Failed to parse JSON record: " + key + ": " + err);
+						callback( err, null );
+						return;
+					}
+					self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? json : null);
+					callback( null, json );
+				}
+			}
+		}); // get
+	},
+	
+	getStream: function(key, callback) {
+		// get readable stream to record value given key
+		var self = this;
+		
+		// The SQLite API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		
+		this.get( key, function(err, buf) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('sqlite', '' + err);
+				return callback(err);
+			}
+			else if (!buf) {
+				// record not found
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback( err, null );
+			}
+			
+			var stream = new BufferStream(buf);
+			callback(null, stream, { mod: 1, len: buf.length });
+		} );
+	},
+	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		var self = this;
+		
+		// The SQLite API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		
+		this.get( key, function(err, buf) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('sqlite', '' + err);
+				return callback(err);
+			}
+			else if (!buf) {
+				// record not found
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback( err, null );
+			}
+			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
+				return;
+			}
+			
+			var range = buf.slice(start, end + 1);
+			var stream = new BufferStream(range);
+			callback(null, stream, { mod: 1, len: buf.length });
+		} );
+	},
+	
+	delete: function(key, callback) {
+		// delete SQLite key given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Deleting SQLite Object: " + key);
+		
+		this.commands.delete.run({ $key: key }, function(err) {
+			// In sqlite3 callbacks `this` is special, and contains `changes`
+			if (!err && !this.changes) {
+				err = new Error("Not found");
+				err.code = "NoSuchKey";
+			}
+			if (err) {
+				self.logError('sqlite', "Failed to delete object: " + key + ": " + err);
+				err = new Error("Failed to delete object: " + key + ": " + err);
+				return callback(err);
+			}
+			self.logDebug(9, "Delete complete: " + key);
+			callback();
+		}); // delete
+	},
+	
+	runMaintenance: function(callback) {
+		// run daily maintenance
+		callback();
+	},
+	
+	shutdown: function(callback) {
+		// shutdown storage
+		var self = this;
+		this.logDebug(2, "Shutting down SQLite");
+		
+		if (this.db) {
+			// finalize all statements and close db
+			async.eachSeries( Object.keys(this.commands),
+				function(key, callback) {
+					self.commands[key].finalize(callback);
+				},
+				function() {
+					self.db.close( function(err) {
+						if (err) self.logError('sqlite', "Failed to shutdown database cleanly: " + err);
+						else self.logDebug(3, "Shutdown complete");
+						callback();
+					} );
+					self.db = null;
+					self.commands = null;
+				}
+			); // eachSeries
+		}
+	}
+	
+});
+
+// Modified the following snippet from node-streamifier:
+// Copyright (c) 2014 Gabriel Llamas, MIT Licensed
+
+var util = require('util');
+var stream = require('stream');
+
+var BufferStream = function (object, options) {
+	if (object instanceof Buffer || typeof object === 'string') {
+		options = options || {};
+		stream.Readable.call(this, {
+			highWaterMark: options.highWaterMark,
+			encoding: options.encoding
+		});
+	} else {
+		stream.Readable.call(this, { objectMode: true });
+	}
+	this._object = object;
+};
+
+util.inherits(BufferStream, stream.Readable);
+
+BufferStream.prototype._read = function () {
+	this.push(this._object);
+	this._object = null;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.1.0",
+	"version": "3.1.2",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -18,7 +18,8 @@
 		"nosql",
 		"s3",
 		"couchbase",
-		"redis"
+		"redis",
+		"sqlite"
 	],
 	"dependencies": {
 		"@aws-sdk/client-s3": "3.186.0",
@@ -27,16 +28,16 @@
 		"he": "1.1.1",
 		"moo": "0.4.3",
 		"nearley": "2.20.1",
-		"pixl-cache": "^1.0.1",
+		"pixl-cache": "^1.0.6",
 		"pixl-class": "^1.0.3",
-		"pixl-perf": "^1.0.0",
-		"pixl-server": "^1.0.0",
-		"pixl-tools": "^1.0.24",
+		"pixl-perf": "^1.0.9",
+		"pixl-server": "^1.0.39",
+		"pixl-tools": "^1.0.33",
 		"porter-stemmer": "0.9.1",
 		"unidecode": "0.1.8"
 	},
 	"devDependencies": {
-		"pixl-unit": "^1.0.0"
+		"pixl-unit": "^1.0.14"
 	},
 	"scripts": {
 		"test": "pixl-unit test/test.js",

package/storage.js

@@ -670,6 +670,16 @@ module.exports = Class.create({
 			}
 		}; // deleteExpiredRecord
 		
+		var doEngineMaint = function() {
+			// allow engine to run maint as well
+			self.engine.runMaintenance( function() {
+				stats.elapsed_sec = Tools.timeNow() - stats.time_start;
+				self.logDebug(3, "Daily maintenance complete");
+				self.logTransaction('maint', cleanup_list_path, stats);
+				if (callback) callback();
+			} );
+		}; // finish
+		
 		this.listEach( cleanup_list_path, 
 			function(item, item_idx, callback) {
 				// delete item if still expired
@@ -705,7 +715,7 @@ module.exports = Class.create({
 				// list iteration complete
 				if (err) {
 					self.logDebug(10, "Failed to load list, skipping maintenance (probably harmless)", cleanup_list_path);
-					if (callback) callback();
+					doEngineMaint();
 				}
 				else {
 					// no error, delete list
@@ -713,15 +723,7 @@ module.exports = Class.create({
 						if (err) {
 							self.logError('maint', "Failed to delete cleanup list: " + cleanup_list_path + ": " + err);
 						}
-						
-						// allow engine to run maint as well
-						self.engine.runMaintenance( function() {
-							stats.elapsed_sec = Tools.timeNow() - stats.time_start;
-							self.logDebug(3, "Daily maintenance complete");
-							self.logTransaction('maint', cleanup_list_path, stats);
-							if (callback) callback();
-						} );
-						
+						doEngineMaint();
 					} ); // listDelete
 				} // succes
 			} // list complete

package/test/config.json

@@ -64,6 +64,10 @@
 			"host": "127.0.0.1",
 			"port": 6379,
 			"keyPrefix": ""
+		},
+		"SQLite": {
+			"base_dir": "data",
+			"filename": "sqlite.db"
 		}
 	}
 }

package/test/test-main.js

@@ -450,7 +450,7 @@ module.exports = {
 					test.ok( !!storageStream, "Got storage stream as 2nd arg");
 					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
 					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
-					test.ok( streamInfo.len == 6, "Info has correct data length");
+					test.ok( streamInfo.len == 43, "Info has correct data length (expected 43, got " + streamInfo.len + ")");
 					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
 					
 					outStream.on('finish', function() {
@@ -497,7 +497,7 @@ module.exports = {
 					test.ok( !!storageStream, "Got storage stream as 2nd arg");
 					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
 					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
-					test.ok( streamInfo.len == 23, "Info has correct data length");
+					test.ok( streamInfo.len == 43, "Info has correct data length (expected 43, got " + streamInfo.len + ")");
 					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
 					
 					outStream.on('finish', function() {
@@ -544,7 +544,7 @@ module.exports = {
 					test.ok( !!storageStream, "Got storage stream as 2nd arg");
 					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
 					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
-					test.ok( streamInfo.len == 10, "Info has correct data length");
+					test.ok( streamInfo.len == 43, "Info has correct data length (expected 43, got " + streamInfo.len + ")");
 					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
 					
 					outStream.on('finish', function() {