pixl-server-storage 4.0.2 → 4.1.0

package/README.md

@@ -396,8 +396,8 @@ To enable the filesystem cache, include a `cache` object in your `Filesystem` co
 		"base_dir": "/let/data/myserver",
 		"cache": {
 			"enabled": true,
-			"maxItems": 1000,
-			"maxBytes": 10485760
+			"maxItems": 100000,
+			"maxBytes": 104857600
 		}
 	}
 }
@@ -440,8 +440,8 @@ If you want to use [Amazon S3](http://aws.amazon.com/s3/) as a backing store, co
 		},
 		"cache": {
 			"enabled": true,
-			"maxItems": 1000,
-			"maxBytes": 10485760
+			"maxItems": 100000,
+			"maxBytes": 104857600
 		}
 	}
 }
@@ -498,8 +498,8 @@ To enable the S3 cache, include a `cache` object in your `S3` configuration with
 {
 	"cache": {
 		"enabled": true,
-		"maxItems": 1000,
-		"maxBytes": 10485760
+		"maxItems": 100000,
+		"maxBytes": 104857600
 	}
 }
 ```
@@ -564,8 +564,8 @@ Here is a complete example with the new parameters added:
 			},
 			"cache": {
 				"enabled": true,
-				"maxItems": 1000,
-				"maxBytes": 10485760
+				"maxItems": 100000,
+				"maxBytes": 104857600
 			}
 		}
 	}
@@ -668,6 +668,11 @@ Then configure your storage thusly:
 			"compress": true,
 			"keep": 7
 		},
+		"cache": {
+			"enabled": true,
+			"maxItems": 100000,
+			"maxBytes": 104857600
+		},
 		"keyPrefix": "",
 		"keyTemplate": ""
 	}
@@ -680,10 +685,61 @@ The optional `pragmas` object allows you set one or more [SQLite Pragmas](https:
 
 The optional `backups` object allows you to configure daily backups of the SQLite database file, which happens during the [daily maintenance](#daily-maintenance) event.  Set `enabled` to `true` to enable backups, then set `dir` to the directory to hold the backups, `filename` to the destination filename (you can use date/time placeholders here), `compress` to compress the backups with gzip, and `keep` to keep the latest N backups in the dir (i.e. auto-delete the oldest ones).  Please note that the backup operation locks the database, so it will cause other storage operations to hang while it is writing to the file.
 
+See [S3 Cache](#s3-cache) for details on the `cache` section, as it works in the same way.
+
 The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key-prefix) feature.  It allows you to prefix all the SQLite keys with a common string, to separate your application's data in a shared database situation.
 
 The optional `keyTemplate` property works similarly to the [S3 Key Template](#s3-key-template) feature.  It allows you to specify an exact layout of MD5 hash characters, which can be prefixed, mixed in with or postfixed after the key.
 
+## Postgres
+
+If you want to use [Postgres](https://www.postgresql.org/) as a backing store, here is how to do so.  First, you need to manually install the [pg](https://www.npmjs.com/package/pg) module into your app:
+
+```sh
+npm install --save pg
+```
+
+Then configure your storage thusly:
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"min": 0,
+		"max": 32,
+		"host": "localhost",
+		"database": "postgres",
+		"user": "postgres",
+		"password": "postgres",
+		"port": 5432,
+		"statement_timeout": 5000,
+		"query_timeout": 6000,
+		"connectionTimeoutMillis": 30000,
+		"idleTimeoutMillis": 10000,
+		"table": "items",
+		"cache": {
+			"enabled": true,
+			"maxItems": 100000,
+			"maxBytes": 104857600
+		}
+	}
+}
+```
+
+The Postgres engine uses a single table to act as the key/value store.  The table is automatically created on startup if needed:
+
+```sql
+CREATE TABLE IF NOT EXISTS items( key TEXT PRIMARY KEY, value BYTEA NOT NULL, modified BIGINT NOT NULL )
+```
+
+You can customize the table name via the `table` configuration property.  It defaults to simply `items`.
+
+[Connection Pooling](https://node-postgres.com/features/pooling) is utilized to open many simultaneous connections to greatly increase throughput.  Set the `max` property to the maximum number of concurrent connections your database can handle.  Ideally this should be set to the global [concurrency](#concurrency) setting.
+
+See [S3 Cache](#s3-cache) for details on the `cache` section, as it works in the same way.
+
+Note that pooling and caching have little effect when the database is running locally (i.e. localhost).  They offer much more benefit when the database is located remotely over some number of network hops.
+
 ## Hybrid
 
 Your application may need the features of multiple engines.  Specifically, you may want JSON (document) records to use one engine, and binary records to use another.  Binary records are specified with keys that end in a file extension, e.g. `.jpg`.  To facilitate this, there is a `Hybrid` engine available, which can load multiple sub-engines, one for JSON keys and one for binary keys.  Example use:

package/engines/Couchbase.js

@@ -288,7 +288,6 @@ module.exports = Class.create({
 				end = buf.length ? buf.length - 1 : 0;
 			}
 			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
-				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
 				return;
 			}

package/engines/Filesystem.js

@@ -569,7 +569,6 @@ module.exports = Class.create({
 				end = stats.size ? stats.size - 1 : 0;
 			}
 			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= stats.size) || (end < start) || (end >= stats.size)) {
-				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + stats.size + ")"), null );
 				return;
 			}

package/engines/Postgres.js

@@ -0,0 +1,475 @@
+// Postgres Storage Plugin
+// Copyright (c) 2026 Joseph Huckaby
+// Released under the MIT License
+
+// Requires the 'pg' module
+
+const fs = require('fs');
+const Path = require('path');
+const zlib = require('zlib');
+const Class = require("pixl-class");
+const Component = require("pixl-server/component");
+const pg = require('pg');
+const Tools = require("pixl-tools");
+const Perf = require("pixl-perf");
+const Cache = require("pixl-cache");
+const async = require('async');
+const noop = function() {};
+
+module.exports = Class.create({
+	
+	__name: 'Postgres',
+	__parent: Component,
+	
+	defaultConfig: {
+		min: 0,
+		max: 32,
+		host: "localhost",
+		database: "",
+		user: "",
+		password: "",
+		port: 5432,
+		statement_timeout: 5000,
+		query_timeout: 6000,
+		connectionTimeoutMillis: 30000,
+		idleTimeoutMillis: 10000,
+		table: "items"
+	},
+	
+	startup: function(callback) {
+		// setup Postgres connection
+		var self = this;
+		this.logDebug(2, "Setting up Postgres", Tools.copyHashRemoveKeys( this.config.get(), { password:1 }) );
+		this.setup(callback);
+	},
+	
+	setup: function(callback) {
+		// setup Postgres connection
+		var self = this;
+		var pg_config = this.config.get();
+		
+		this.commands = {
+			create: `CREATE TABLE IF NOT EXISTS ${pg_config.table}( key TEXT PRIMARY KEY, value BYTEA NOT NULL, modified BIGINT NOT NULL )`,
+			get: `SELECT modified, value FROM ${pg_config.table} WHERE key = $key LIMIT 1`,
+			head: `SELECT modified, octet_length(value) AS len FROM ${pg_config.table} WHERE key = $key LIMIT 1`,
+			put: `INSERT INTO ${pg_config.table} (key, value, modified) VALUES ($key, $value, $now) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, modified = EXCLUDED.modified`,
+			delete: `DELETE FROM ${pg_config.table} WHERE key = $key`
+		};
+		
+		// optional LRU cache
+		this.cache = null;
+		var cache_opts = pg_config.cache;
+		if (cache_opts && cache_opts.enabled) {
+			this.logDebug(3, "Setting up LRU cache", cache_opts);
+			this.cache = new Cache( Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) );
+			this.cache.on('expire', function(item, reason) {
+				self.logDebug(9, "Expiring LRU cache object: " + item.key + " due to: " + reason, {
+					key: item.key,
+					reason: reason,
+					totalCount: self.cache.count,
+					totalBytes: self.cache.bytes
+				});
+			});
+		}
+		
+		// pass entire config to pg.Pool, sans our custom keys
+		var db = this.db = new pg.Pool( 
+			Tools.copyHashRemoveKeys( pg_config, { cache: 1, table: 1 }) 
+		);
+		
+		db.on('error', function(err) {
+			self.logError('db', "DB Error: " + err, err);
+			if (callback) { callback(err); callback = null; }
+		});
+		db.on('connect', function(client) {
+			self.logDebug(9, "Opened DB pool connection");
+		});
+		db.on('remove', function(client) {
+			self.logDebug(9, "Closed DB pool connection");
+		});
+		
+		// create initial table
+		this.logDebug(3, "Creating initial table: " + pg_config.table);
+		this.query( this.commands.create, {}, function(err, res) {
+			if (err) self.logError('db', "Failed to create initial table: " + err, err);
+			else self.logDebug(3, "Setup complete");
+			if (callback) { callback(err); callback = null; }
+		} );
+	},
+	
+	query(sql, args, callback) {
+		// run and log db query, and error if applicable
+		// allow for named variables in sql, expand to numbered $ variables for pg
+		var self = this;
+		
+		// perform $ placeholder substitution to populate sargs array
+		var sargs = [];
+		sql = sql.replace(/\$(\w+)/g, function(m_all, name) {
+			var value = args[name];
+			sargs.push(value);
+			value = '$' + sargs.length;
+			return value;
+		} );
+		
+		this.db.query( sql, sargs, function(err, res) {
+			if (err) {
+				self.logError('db', "Failed to execute SQL: " + err, { sql, err });
+				return callback(err);
+			}
+			
+			self.logDebug(9, "SQL query completed successfully", {
+				command: res.command || '',
+				rowCount: res.rowCount || 0
+			});
+			
+			callback(null, res);
+		} ); // db.query
+	},
+	
+	prepKey: function(key) {
+		// prepare key (no-op)
+		return key;
+	},
+	
+	put: function(key, value, callback) {
+		// store key+value in Postgres
+		var self = this;
+		var now = Tools.timeNow(true);
+		key = this.prepKey(key);
+		
+		var is_binary = this.storage.isBinaryKey(key);
+		
+		if (is_binary) {
+			this.logDebug(9, "Storing Postgres Binary Object: " + key, '' + value.length + ' bytes');
+		}
+		else {
+			this.logDebug(9, "Storing Postgres JSON Object: " + key, this.debugLevel(10) ? value : null);
+			value = Buffer.from( JSON.stringify( value ) );
+		}
+		
+		this.query( this.commands.put, { key: key, value: value, now: now }, function(err, res) {
+			if (err) {
+				err.message = "Failed to store object: " + key + ": " + err;
+				self.logError('pg', '' + err);
+				if (callback) callback(err);
+				return;
+			}
+			
+			self.logDebug(9, "Store complete: " + key);
+			// possibly cache in LRU
+			if (self.cache && !is_binary) {
+				self.cache.set( key, value, { date: now } );
+			}
+			if (callback) callback(null);
+		} ); // query
+	},
+	
+	putStream: function(key, inp, callback) {
+		// store key+value in Postgres using read stream
+		var self = this;
+		
+		// The Postgres API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		
+		var chunks = [];
+		var done = false;
+		var finish = function(err) {
+			if (done) return;
+			done = true;
+			callback(err);
+		};
+		
+		inp.on('error', function(err) {
+			err.message = "Failed to store stream: " + key + ": " + err;
+			self.logError('pg', '' + err);
+			finish(err);
+		} );
+		inp.on('data', function(chunk) {
+			if (done) return;
+			chunks.push( chunk );
+		} );
+		inp.on('end', function() {
+			if (done) return;
+			var buf = Buffer.concat(chunks);
+			self.put( key, buf, finish );
+		} );
+	},
+	
+	head: function(key, callback) {
+		// head pg item given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		// check cache first
+		if (this.cache && this.cache.has(key)) {
+			var item = this.cache.getMeta(key);
+			
+			process.nextTick( function() {
+				self.logDebug(9, "Cached head complete: " + key);
+				callback( null, {
+					mod: item.date,
+					len: item.value.length
+				} );
+			} );
+			return;
+		} // cache
+		
+		this.query( this.commands.head, { key: key }, function(err, res) {
+			if (err) {
+				err.message = "Failed to head key: " + key + ": " + err;
+				self.logError('pg', '' + err);
+				callback(err);
+				return;
+			}
+			
+			var row = res.rows[0];
+			
+			if (!row) {
+				var err = new Error("Failed to head key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback(err, null);
+			}
+			callback(null, { mod: parseInt(row.modified, 10), len: row.len });
+		} ); // query
+	},
+	
+	get: function(key, callback) {
+		// fetch Postgres value given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		var is_binary = this.storage.isBinaryKey(key);
+		
+		// check cache first
+		if (this.cache && !is_binary && this.cache.has(key)) {
+			var data = this.cache.get(key);
+			
+			process.nextTick( function() {
+				try { data = JSON.parse( data.toString() ); }
+				catch (e) {
+					self.logError('pg', "Failed to parse JSON record: " + key + ": " + e);
+					callback( e, null );
+					return;
+				}
+				self.logDebug(9, "Cached JSON fetch complete: " + key, self.debugLevel(10) ? data : null);
+				
+				callback( null, data );
+			} );
+			return;
+		} // cache
+		
+		this.logDebug(9, "Fetching Postgres Object: " + key);
+		
+		this.query( this.commands.get, { key: key }, function(err, res) {
+			if (err) {
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('pg', '' + err);
+				callback(err);
+				return;
+			}
+			
+			var row = res.rows[0];
+			
+			if (!row) {
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback(err, null);
+			}
+			if (is_binary) {
+				self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
+				return callback(null, row.value);
+			}
+			if (self.cache) {
+				self.cache.set( key, row.value, { date: parseInt(row.modified, 10) } );
+			}
+			var json = null;
+			try { json = JSON.parse( row.value.toString() ); }
+			catch (err) {
+				self.logError('pg', "Failed to parse JSON record: " + key + ": " + err);
+				return callback(err, null);
+			}
+			self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? json : null);
+			callback(null, json);
+		} ); // query
+	},
+	
+	getBuffer: function(key, callback) {
+		// fetch Postgres buffer given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Fetching Postgres Object: " + key);
+		
+		this.query( this.commands.get, { key: key }, function(err, res) {
+			if (err) {
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('pg', '' + err);
+				callback(err);
+				return;
+			}
+			
+			var row = res.rows[0];
+			
+			if (!row) {
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback(err, null);
+			}
+			self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
+			callback(null, row.value);
+		} ); // query
+	},
+	
+	getStream: function(key, callback) {
+		// get readable stream to record value given key
+		var self = this;
+		
+		// The Postgres API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		
+		this.query( this.commands.get, { key: this.prepKey(key) }, function(err, res) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('pg', '' + err);
+				return callback(err);
+			}
+			
+			var row = res.rows[0];
+			
+			if (!row) {
+				// record not found
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback( err, null );
+			}
+			
+			self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
+			var stream = new BufferStream(row.value);
+			callback(null, stream, { mod: parseInt(row.modified, 10), len: row.value.length });
+		} ); // query
+	},
+	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		var self = this;
+		
+		// The Postgres API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		
+		this.query( this.commands.get, { key: this.prepKey(key) }, function(err, res) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('pg', '' + err);
+				return callback(err);
+			}
+			
+			var row = res.rows[0];
+			
+			if (!row) {
+				// record not found
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback( err, null );
+			}
+			
+			var buf = row.value;
+			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
+				return;
+			}
+			
+			var range = buf.slice(start, end + 1);
+			var stream = new BufferStream(range);
+			callback(null, stream, { mod: parseInt(row.modified, 10), len: buf.length });
+		} ); // query
+	},
+	
+	delete: function(key, callback) {
+		// delete Postgres key given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Deleting Postgres Object: " + key);
+		
+		this.query( this.commands.delete, { key: key }, function(err, res) {
+			if (err) {
+				self.logError('pg', "Failed to delete object: " + key + ": " + err);
+				callback(err);
+				return;
+			}
+			
+			if (!res.rowCount) {
+				var err = new Error("Failed to delete object: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback(err);
+			}
+			self.logDebug(9, "Delete complete: " + key);
+			if (self.cache && self.cache.has(key)) {
+				self.cache.delete(key);
+			}
+			callback();
+		} ); // query
+	},
+	
+	runMaintenance: function(callback) {
+		// run daily maintenance
+		callback();
+	},
+	
+	shutdown: function(callback) {
+		// shutdown storage
+		var self = this;
+		this.logDebug(2, "Shutting down Postgres");
+		
+		if (this.db) {
+			this.db.end( function(err) {
+				if (err) self.logError('pg', "Failed to shutdown database cleanly: " + err);
+				else self.logDebug(3, "Shutdown complete");
+				delete self.db;
+				callback();
+			} );
+		}
+		else {
+			this.logDebug(3, "Shutdown complete");
+			callback();
+		}
+	}
+	
+});
+
+// Modified the following snippet from node-streamifier:
+// Copyright (c) 2014 Gabriel Llamas, MIT Licensed
+
+var util = require('util');
+var stream = require('stream');
+
+var BufferStream = function (object, options) {
+	if (object instanceof Buffer || typeof object === 'string') {
+		options = options || {};
+		stream.Readable.call(this, {
+			highWaterMark: options.highWaterMark,
+			encoding: options.encoding
+		});
+	} else {
+		stream.Readable.call(this, { objectMode: true });
+	}
+	this._object = object;
+};
+
+util.inherits(BufferStream, stream.Readable);
+
+BufferStream.prototype._read = function () {
+	this.push(this._object);
+	this._object = null;
+};

package/engines/Redis.js

@@ -289,7 +289,6 @@ module.exports = Class.create({
 				end = buf.length ? buf.length - 1 : 0;
 			}
 			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
-				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
 				return;
 			}

package/engines/RedisCluster.js

@@ -303,7 +303,6 @@ module.exports = Class.create({
 				end = buf.length ? buf.length - 1 : 0;
 			}
 			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
-				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
 				return;
 			}

package/engines/SQLite.js

@@ -153,6 +153,7 @@ module.exports = Class.create({
 				err.message = "Failed to store object: " + key + ": " + err;
 				self.logError('sqlite', '' + err);
 				if (callback) callback(err);
+				return;
 			}
 			
 			self.logDebug(9, "Store complete: " + key);
@@ -208,7 +209,7 @@ module.exports = Class.create({
 			catch (err) {
 				err.message = "Failed to head key: " + key + ": " + err;
 				self.logError('sqlite', '' + err);
-				callback(err);
+				return callback(err);
 			}
 			
 			if (!row) {
@@ -256,6 +257,7 @@ module.exports = Class.create({
 				err.message = "Failed to fetch key: " + key + ": " + err;
 				self.logError('sqlite', '' + err);
 				callback(err);
+				return;
 			}
 			
 			if (!row) {
@@ -297,6 +299,7 @@ module.exports = Class.create({
 				err.message = "Failed to fetch key: " + key + ": " + err;
 				self.logError('sqlite', '' + err);
 				callback(err);
+				return;
 			}
 			
 			if (!row) {
@@ -365,7 +368,6 @@ module.exports = Class.create({
 				end = buf.length ? buf.length - 1 : 0;
 			}
 			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
-				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
 				return;
 			}
@@ -391,6 +393,7 @@ module.exports = Class.create({
 			catch (err) {
 				self.logError('sqlite', "Failed to delete object: " + key + ": " + err);
 				callback(err);
+				return;
 			}
 			
 			if (!info.changes) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "4.0.2",
+	"version": "4.1.0",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -34,7 +34,7 @@
 		"pixl-class": "^1.0.3",
 		"pixl-perf": "^1.0.9",
 		"pixl-server": "^1.0.39",
-		"pixl-tools": "^1.0.33",
+		"pixl-tools": "^2.0.2",
 		"porter-stemmer": "0.9.1",
 		"stream-meter": "1.0.4",
 		"unidecode": "0.1.8"

package/test/config.json

@@ -84,6 +84,25 @@
 				"compress": true,
 				"keep": 7
 			}
+		},
+		"Postgres": {
+			"min": 0,
+			"max": 32,
+			"host": "localhost",
+			"database": "postgres",
+			"user": "postgres",
+			"password": "postgres",
+			"port": 5432,
+			"statement_timeout": 5000,
+			"query_timeout": 6000,
+			"connectionTimeoutMillis": 30000,
+			"idleTimeoutMillis": 10000,
+			"table": "items",
+			"cache": {
+				"enabled": true,
+				"maxItems": 100000,
+				"maxBytes": 104857600
+			}
 		}
 	}
 }