pixl-server-storage 3.2.1 → 3.2.3

package/README.md

@@ -645,6 +645,13 @@ Then configure your storage thusly:
 			"cache_size": -100000,
 			"journal_mode": "WAL"
 		},
+		"backups": {
+			"enabled": false,
+			"dir": "data/backups",
+			"filename": "backup-[yyyy]-[mm]-[dd].db",
+			"compress": true,
+			"keep": 7
+		},
 		"keyPrefix": "",
 		"keyTemplate": ""
 	}
@@ -655,6 +662,8 @@ The `base_dir` defaults to the current working directory, and will be created on
 
 The optional `pragmas` object allows you set one or more [SQLite Pragmas](https://www.sqlite.org/pragma.html#toc) (configuration settings) on the database at startup.  Here you can specify things such as [auto_vacuum](https://www.sqlite.org/pragma.html#pragma_auto_vacuum), [cache_size](https://www.sqlite.org/pragma.html#pragma_cache_size) and [journal_mode](https://www.sqlite.org/pragma.html#pragma_journal_mode), among many others.
 
+The optional `backups` object allows you to configure daily backups of the SQLite database file, which happens during the [daily maintenance](#daily-maintenance) event.  Set `enabled` to `true` to enable backups, then set `dir` to the directory to hold the backups, `filename` to the destination filename (you can use date/time placeholders here), `compress` to compress the backups with gzip, and `keep` to keep the latest N backups in the dir (i.e. auto-delete the oldest ones).  Please note that the backup operation locks the database, so it will cause other storage operations to hang while it is writing to the file.
+
 The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key-prefix) feature.  It allows you to prefix all the SQLite keys with a common string, to separate your application's data in a shared database situation.
 
 The optional `keyTemplate` property works similarly to the [S3 Key Template](#s3-key-template) feature.  It allows you to specify an exact layout of MD5 hash characters, which can be prefixed, mixed in with or postfixed after the key.

package/engines/SQLite.js

@@ -5,12 +5,16 @@
 // Requires the 'sqlite3' module from npm
 // npm install --save sqlite3
 
+const fs = require('fs');
 const Path = require('path');
+const zlib = require('zlib');
 const Class = require("pixl-class");
 const Component = require("pixl-server/component");
 const SQLite3 = require('sqlite3');
 const Tools = require("pixl-tools");
+const Perf = require("pixl-perf");
 const async = require('async');
+const noop = function() {};
 
 module.exports = Class.create({
 	
@@ -348,7 +352,138 @@ module.exports = Class.create({
 	
 	runMaintenance: function(callback) {
 		// run daily maintenance
-		callback();
+		// config.backups: { enabled, dir, filename, compress?, keep? }
+		var self = this;
+		var backups = this.config.get('backups') || null;
+		if (!backups || !backups.enabled || !backups.dir || !backups.filename) return callback();
+		
+		// here be dragons: try to wait for a moment BETWEEN transaction commits, to minimize risk of an "inconsistent" backup
+		if (this.storage.transactions && Tools.findObject(Object.values(this.storage.transactions), { committing: true })) {
+			// wait for next commitEnd event and try again
+			this.logDebug(9, "Maintenance: Transaction commit(s) are in progress, waiting until next commitEnd event...");
+			
+			this.storage.once('commitEnd', function(trans) {
+				self.logDebug(9, "Maintenance: Transaction commit " + trans.id + " (" + trans.path + ") ended, retrying now...");
+				self.runMaintenance(callback);
+			});
+			
+			return;
+		}
+		
+		// track perf for logging
+		var perf = new Perf();
+		perf.begin();
+		perf.begin('prep');
+		
+		// allow dir & filename to have date/time placeholders, e.g. `backup-[yyyy]-[mm]-[dd].db`
+		var dargs = Tools.getDateArgs( Tools.timeNow() );
+		var file = Tools.sub( Path.join(backups.dir, backups.filename), dargs, true );
+		if (!file) {
+			this.logError('backup', "Failed to expand placeholders on backup file: " + backups.filename);
+			return callback();
+		}
+		
+		// strip .gz prefix, just in case the user added it
+		file = file.replace(/\.gz$/i, '');
+		
+		// auto-create parent dirs as needed
+		if (!fs.existsSync(backups.dir)) {
+			try { Tools.mkdirp.sync(backups.dir); }
+			catch (err) {
+				this.logError('backup', "Failed to create backup directory: " + backups.dir);
+				return callback();
+			}
+		}
+		
+		// here we go
+		if (fs.existsSync(file)) fs.unlinkSync(file);
+		this.logDebug(6, "Performing database backup to: " + file);
+		perf.end('prep');
+		
+		var finish = function() {
+			// all done
+			perf.end();
+			self.logDebug(6, "Maintenance complete", perf.metrics());
+			callback();
+		};
+		
+		var keep = function() {
+			// delete oldest backups if over N on disk
+			var files = Tools.findFilesSync( backups.dir, { stats: true } );
+			if (files.length <= backups.keep) return finish();
+			perf.begin('keep');
+			
+			self.logDebug(6, "There are " + files.length + " backup files in " + backups.dir + ", keeping the latest " + backups.keep);
+			
+			// sort mtime descending so the newest are at the start of the array, then splice those off, leaving only the oldest to delete
+			Tools.sortBy( files, 'mtime', { type: 'number', dir: -1 } );
+			files.splice( 0, backups.keep );
+			
+			files.forEach( function(file) {
+				self.logDebug(7, "Deleting old backup file: " + file.path, file);
+				try { fs.unlinkSync(file.path); } catch (err) {
+					self.logError('backup', "Failed to delete backup file: " + file.path + ": " + err);
+				}
+			} );
+			
+			perf.end('keep');
+			finish();
+		}; // keep
+		
+		var compress = function() {
+			// compress backup and delete original
+			var gz_file = file + '.gz';
+			perf.begin('compress');
+			
+			if (fs.existsSync(gz_file)) fs.unlinkSync(gz_file);
+			self.logDebug(6, "Compressing backup to: " + gz_file);
+			
+			var gzip = zlib.createGzip();
+			var inp = fs.createReadStream( file );
+			var outp = fs.createWriteStream( gz_file );
+			
+			var handleError = function(err) {
+				self.logError('backup', "Failed to compress backup: " + gz_file + ": " + err);
+				fs.unlink(gz_file, noop);
+				
+				perf.end('compress');
+				if (backups.keep) keep();
+				else finish();
+			};
+			
+			gzip.on('error', handleError);
+			inp.on('error', handleError);
+			outp.on('error', handleError);
+			
+			inp.pipe(gzip).pipe(outp).on('finish', function() {
+				self.logDebug(6, "Backup compression complete: " + gz_file);
+				fs.unlink(file, noop);
+				
+				perf.end('compress');
+				if (backups.keep) keep();
+				else finish();
+			} );
+		}; // compress
+		
+		// Perform the backup -- unfortunately this locks the DB, and we can't use progressive mode
+		// because of how we handle transactions "outside" of SQLite's control.
+		perf.begin('backup');
+		
+		this.db.run('VACUUM INTO $filename', { $filename: Path.resolve(file) }, function(err) {
+			perf.end('backup');
+			if (err) {
+				self.logError('backup', "SQLite backup operation failed: " + err);
+				fs.unlink(file, noop);
+				return finish();
+			}
+			
+			self.logDebug(6, "SQLite backup operation completed: " + file);
+			
+			// now compress or keep or done
+			if (backups.compress) compress();
+			else if (backups.keep) keep();
+			else finish();
+		});
 	},
 	
 	shutdown: function(callback) {

package/index_types/Number.js

@@ -83,7 +83,7 @@ module.exports = Class.create({
 	
 	filterWords_number: function(value) {
 		// filter number queries
-		value = value.replace(/[^\d\-]+/g, '').replace(/\-/, 'N');
+		value = value.replace(/\.\d+$/, '').replace(/[^\d\-]+/g, '').replace(/\-/, 'N');
 		return value;
 	},
 	

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.2.1",
+	"version": "3.2.3",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -22,9 +22,9 @@
 		"sqlite"
 	],
 	"dependencies": {
-		"@aws-sdk/client-s3": "3.414.0",
-		"@aws-sdk/lib-storage": "3.414.0",
-		"@smithy/node-http-handler": "2.1.4",
+		"@aws-sdk/client-s3": "3.621.0",
+		"@aws-sdk/lib-storage": "3.621.0",
+		"@smithy/node-http-handler": "3.1.4",
 		"async": "2.6.4",
 		"fast-stream-to-buffer": "1.0.0",
 		"he": "1.1.1",

package/storage.js

@@ -721,7 +721,7 @@ module.exports = Class.create({
 		var doEngineMaint = function() {
 			// allow engine to run maint as well
 			self.engine.runMaintenance( function() {
-				stats.elapsed_sec = Tools.timeNow() - stats.time_start;
+				stats.elapsed_sec = Tools.shortFloat( Tools.timeNow() - stats.time_start );
 				self.logDebug(3, "Daily maintenance complete");
 				self.logTransaction('maint', cleanup_list_path, stats);
 				if (callback) callback();

package/transaction.js

@@ -648,6 +648,7 @@ module.exports = Class.create({
 					self.logDebug(3, "Transaction rollback complete: " + trans.id, { path: path });
 					
 					// unlock at the VERY end, as a new transaction may be waiting on the same path
+					self.emit('commitEnd', trans, "rollback");
 					self.unlock( 'C|'+path );
 					self._transUnlock(path);
 					
@@ -787,9 +788,9 @@ module.exports = Class.create({
 				var elapsed = pf.end();
 				
 				if (err) {
+					// Note: in this case unlocking happens in abortTransaction
 					var msg = "Failed to commit transaction: " + path + ": " + err.message;
 					self.logError('commit', msg, { id: trans.id });
-					self.emit('commitEnd', trans, err);
 					return callback( new Error(msg) );
 				}