pixl-server-storage 4.0.3 → 4.1.1

package/README.md

@@ -58,6 +58,7 @@ Here is the table of contents for this current document:
 	* [Redis](#redis)
 		+ [RedisCluster](#rediscluster)
 	* [SQLite](#sqlite)
+	* [Postgres](#postgres)
 	* [Hybrid](#hybrid)
 - [Key Normalization](#key-normalization)
 - [Basic Functions](#basic-functions)
@@ -396,8 +397,8 @@ To enable the filesystem cache, include a `cache` object in your `Filesystem` co
 		"base_dir": "/let/data/myserver",
 		"cache": {
 			"enabled": true,
-			"maxItems": 1000,
-			"maxBytes": 10485760
+			"maxItems": 100000,
+			"maxBytes": 104857600
 		}
 	}
 }
@@ -440,8 +441,8 @@ If you want to use [Amazon S3](http://aws.amazon.com/s3/) as a backing store, co
 		},
 		"cache": {
 			"enabled": true,
-			"maxItems": 1000,
-			"maxBytes": 10485760
+			"maxItems": 100000,
+			"maxBytes": 104857600
 		}
 	}
 }
@@ -498,8 +499,8 @@ To enable the S3 cache, include a `cache` object in your `S3` configuration with
 {
 	"cache": {
 		"enabled": true,
-		"maxItems": 1000,
-		"maxBytes": 10485760
+		"maxItems": 100000,
+		"maxBytes": 104857600
 	}
 }
 ```
@@ -564,8 +565,8 @@ Here is a complete example with the new parameters added:
 			},
 			"cache": {
 				"enabled": true,
-				"maxItems": 1000,
-				"maxBytes": 10485760
+				"maxItems": 100000,
+				"maxBytes": 104857600
 			}
 		}
 	}
@@ -589,7 +590,12 @@ Then configure your storage thusly:
 		"host": "127.0.0.1",
 		"port": 6379,
 		"keyPrefix": "",
-		"keyTemplate": ""
+		"keyTemplate": "",
+		"cache": {
+			"enabled": true,
+			"maxItems": 100000,
+			"maxBytes": 104857600
+		}
 	}
 }
 ```
@@ -600,6 +606,8 @@ The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key
 
 The optional `keyTemplate` property works similarly to the [S3 Key Template](#s3-key-template) feature.  It allows you to specify an exact layout of MD5 hash characters, which can be prefixed, mixed in with or postfixed after the key.
 
+See [S3 Cache](#s3-cache) for details on the `cache` section, as it works in the same way.
+
 ### RedisCluster
 
 If you want to use a Redis cluster (e.g. [AWS ElastiCache](https://aws.amazon.com/elasticache/)), then here is how to do that.  First, you will need to manually install the following module into your app:
@@ -625,7 +633,12 @@ Then configure your storage thusly:
 			}
 		},
 		"keyPrefix": "",
-		"keyTemplate": ""
+		"keyTemplate": "",
+		"cache": {
+			"enabled": true,
+			"maxItems": 100000,
+			"maxBytes": 104857600
+		}
 	}
 }
 ```
@@ -640,6 +653,8 @@ The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key
 
 The optional `keyTemplate` property works similarly to the [S3 Key Template](#s3-key-template) feature.  It allows you to specify an exact layout of MD5 hash characters, which can be prefixed, mixed in with or postfixed after the key.
 
+See [S3 Cache](#s3-cache) for details on the `cache` section, as it works in the same way.
+
 ## SQLite
 
 If you want to use [SQLite](https://sqlite.com/) as a backing store, here is how to do so.  First, you need to manually install the [better-sqlite3](https://www.npmjs.com/package/better-sqlite3) module into your app:
@@ -668,6 +683,11 @@ Then configure your storage thusly:
 			"compress": true,
 			"keep": 7
 		},
+		"cache": {
+			"enabled": true,
+			"maxItems": 100000,
+			"maxBytes": 104857600
+		},
 		"keyPrefix": "",
 		"keyTemplate": ""
 	}
@@ -680,10 +700,61 @@ The optional `pragmas` object allows you set one or more [SQLite Pragmas](https:
 
 The optional `backups` object allows you to configure daily backups of the SQLite database file, which happens during the [daily maintenance](#daily-maintenance) event.  Set `enabled` to `true` to enable backups, then set `dir` to the directory to hold the backups, `filename` to the destination filename (you can use date/time placeholders here), `compress` to compress the backups with gzip, and `keep` to keep the latest N backups in the dir (i.e. auto-delete the oldest ones).  Please note that the backup operation locks the database, so it will cause other storage operations to hang while it is writing to the file.
 
+See [S3 Cache](#s3-cache) for details on the `cache` section, as it works in the same way.
+
 The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key-prefix) feature.  It allows you to prefix all the SQLite keys with a common string, to separate your application's data in a shared database situation.
 
 The optional `keyTemplate` property works similarly to the [S3 Key Template](#s3-key-template) feature.  It allows you to specify an exact layout of MD5 hash characters, which can be prefixed, mixed in with or postfixed after the key.
 
+## Postgres
+
+If you want to use [Postgres](https://www.postgresql.org/) as a backing store, here is how to do so.  First, you need to manually install the [pg](https://www.npmjs.com/package/pg) module into your app:
+
+```sh
+npm install --save pg
+```
+
+Then configure your storage thusly:
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"min": 0,
+		"max": 32,
+		"host": "localhost",
+		"database": "postgres",
+		"user": "postgres",
+		"password": "postgres",
+		"port": 5432,
+		"statement_timeout": 5000,
+		"query_timeout": 6000,
+		"connectionTimeoutMillis": 30000,
+		"idleTimeoutMillis": 10000,
+		"table": "items",
+		"cache": {
+			"enabled": true,
+			"maxItems": 100000,
+			"maxBytes": 104857600
+		}
+	}
+}
+```
+
+The Postgres engine uses a single table to act as the key/value store.  The table is automatically created on startup if needed:
+
+```sql
+CREATE TABLE IF NOT EXISTS items( key TEXT PRIMARY KEY, value BYTEA NOT NULL, modified BIGINT NOT NULL )
+```
+
+You can customize the table name via the `table` configuration property.  It defaults to simply `items`.
+
+[Connection Pooling](https://node-postgres.com/features/pooling) is utilized to open many simultaneous connections to greatly increase throughput.  Set the `max` property to the maximum number of concurrent connections your database can handle.  Ideally this should be set to the global [concurrency](#concurrency) setting.
+
+See [S3 Cache](#s3-cache) for details on the `cache` section, as it works in the same way.
+
+Note that pooling and caching have little effect when the database is running locally (i.e. localhost).  They offer much more benefit when the database is located remotely over some number of network hops.
+
 ## Hybrid
 
 Your application may need the features of multiple engines.  Specifically, you may want JSON (document) records to use one engine, and binary records to use another.  Binary records are specified with keys that end in a file extension, e.g. `.jpg`.  To facilitate this, there is a `Hybrid` engine available, which can load multiple sub-engines, one for JSON keys and one for binary keys.  Example use:

package/engines/Couchbase.js

@@ -288,7 +288,6 @@ module.exports = Class.create({
 				end = buf.length ? buf.length - 1 : 0;
 			}
 			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
-				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
 				return;
 			}

package/engines/Filesystem.js

@@ -215,7 +215,7 @@ module.exports = Class.create({
 		}
 		else {
 			this.logDebug(9, "Storing JSON Object: " + key, this.debugLevel(10) ? value : file);
-			value = this.pretty ? JSON.stringify( value, null, "\t" ) : JSON.stringify( value );
+			value = Buffer.from( this.pretty ? JSON.stringify( value, null, "\t" ) : JSON.stringify( value ) );
 		}
 		
 		var dir = path.dirname( file );
@@ -378,13 +378,22 @@ module.exports = Class.create({
 		if (this.cache && this.cache.has(key)) {
 			var item = this.cache.getMeta(key);
 			
-			process.nextTick( function() {
-				self.logDebug(9, "Cached head complete: " + key);
-				callback( null, {
-					mod: item.date,
-					len: item.value.length
+			if (item.value.length == 0) {
+				process.nextTick( function() {
+					var err = new Error("Failed to head key: " + key + ": File not found");
+					err.code = "NoSuchKey";
+					callback( err );
 				} );
-			} );
+			}
+			else {
+				process.nextTick( function() {
+					self.logDebug(9, "Cached head complete: " + key);
+					callback( null, {
+						mod: item.date,
+						len: item.value.length
+					} );
+				} );
+			}
 			return;
 		} // cache
 		
@@ -393,6 +402,11 @@ module.exports = Class.create({
 				if (err.message.match(/ENOENT/)) {
 					err.message = "File not found";
 					err.code = "NoSuchKey";
+					
+					if (self.cache && !self.storage.isBinaryKey(key)) {
+						// store 'empty' stub in cache
+						self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+					}
 				}
 				else {
 					// log fs errors that aren't simple missing files (i.e. I/O errors)
@@ -421,30 +435,42 @@ module.exports = Class.create({
 		
 		// check cache first
 		if (this.cache && !is_binary && this.cache.has(key)) {
-			var data = this.cache.get(key);
+			var item = this.cache.getMeta(key);
 			
-			process.nextTick( function() {
-				try { data = JSON.parse( data ); }
-				catch (e) {
-					self.logError('file', "Failed to parse JSON record: " + key + ": " + e);
-					callback( e, null );
-					return;
-				}
-				self.logDebug(9, "Cached JSON fetch complete: " + key, self.debugLevel(10) ? data : null);
-				
-				callback( null, data );
-			} );
+			if (item.value.length == 0) {
+				process.nextTick( function() {
+					var err = new Error("Failed to fetch key: " + key + ": File not found");
+					err.code = "NoSuchKey";
+					callback( err );
+				} );
+			}
+			else {
+				process.nextTick( function() {
+					var data = null;
+					try { data = JSON.parse( item.value.toString() ); }
+					catch (e) {
+						self.logError('file', "Failed to parse JSON record: " + key + ": " + e);
+						callback( e, null );
+						return;
+					}
+					self.logDebug(9, "Cached JSON fetch complete: " + key, self.debugLevel(10) ? data : null);
+					
+					callback( null, data );
+				} );
+			}
 			return;
 		} // cache
 		
-		var opts = {};
-		if (!this.storage.isBinaryKey(key)) opts = { encoding: 'utf8' };
-		
-		fs.readFile(file, opts, function (err, data) {
+		fs.readFile(file, function (err, data) {
 			if (err) {
 				if (err.message.match(/ENOENT/)) {
 					err.message = "File not found";
 					err.code = "NoSuchKey";
+					
+					if (self.cache && !is_binary) {
+						// store 'empty' stub in cache
+						self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+					}
 				}
 				else {
 					// log fs errors that aren't simple missing files (i.e. I/O errors)
@@ -461,7 +487,7 @@ module.exports = Class.create({
 			}
 			
 			if (!is_binary) {
-				try { data = JSON.parse( data ); }
+				try { data = JSON.parse( data.toString() ); }
 				catch (e) {
 					self.logError('file', "Failed to parse JSON record: " + key + ": " + e);
 					callback( e, null );
@@ -569,7 +595,6 @@ module.exports = Class.create({
 				end = stats.size ? stats.size - 1 : 0;
 			}
 			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= stats.size) || (end < start) || (end >= stats.size)) {
-				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + stats.size + ")"), null );
 				return;
 			}
@@ -596,6 +621,11 @@ module.exports = Class.create({
 				if (err.message.match(/ENOENT/)) {
 					err.message = "File not found";
 					err.code = "NoSuchKey";
+					
+					if (self.cache && !self.storage.isBinaryKey(key)) {
+						// store 'empty' stub in cache
+						self.cache.set( key, Buffer.alloc(0), { date: 0 } );
+					}
 				}
 				
 				self.logError('file', "Failed to delete file: " + key + ": " + file + ": " + err.message);
@@ -606,9 +636,10 @@ module.exports = Class.create({
 			else {
 				self.logDebug(9, "Delete complete: " + key);
 				
-				// possibly delete from LRU cache as well
-				if (self.cache && self.cache.has(key)) {
-					self.cache.delete(key);
+				// possibly "delete" from LRU cache as well
+				if (self.cache && !self.storage.isBinaryKey(key)) {
+					// store 'empty' stub in cache
+					self.cache.set( key, Buffer.alloc(0), { date: 0 } );
 				}
 				
 				// cleanup parent dirs if empty