pixl-server-storage 4.2.1 → 4.2.2

package/README.md

@@ -56,9 +56,14 @@ Here is the table of contents for this current document:
 		+ [S3 Cache](#s3-cache)
 		+ [S3 Compatible Services](#s3-compatible-services)
 	* [Redis](#redis)
+		+ [Redis Auth and TLS](#redis-auth-and-tls)
 		+ [RedisCluster](#rediscluster)
+			- [RedisCluster Auth and TLS](#rediscluster-auth-and-tls)
 	* [SQLite](#sqlite)
 	* [Postgres](#postgres)
+		+ [Postgres SSL](#postgres-ssl)
+		+ [AWS RDS](#aws-rds)
+		+ [Postgres Password Plugin](#postgres-password-plugin)
 	* [Hybrid](#hybrid)
 - [Key Normalization](#key-normalization)
 - [Basic Functions](#basic-functions)
@@ -590,6 +595,8 @@ Then configure your storage thusly:
 	"Redis": {
 		"host": "127.0.0.1",
 		"port": 6379,
+		"username": "",
+		"password": "",
 		"keyPrefix": "",
 		"keyTemplate": "",
 		"cache": {
@@ -603,6 +610,58 @@ Then configure your storage thusly:
 
 Set the `host` and `port` for your own Redis server setup.  Please see [Common Redis Options](https://redis.github.io/ioredis/interfaces/CommonRedisOptions.html) for other things you can include here, such as timeouts, authentication and database selection.
 
+### Redis Auth and TLS
+
+For production, Redis is usually deployed on a private network, firewalled to trusted clients, and protected with authentication as an extra layer of safety.  Modern Redis ACL users use both `username` and `password`.
+
+Configure `username` and `password` like this:
+
+```json
+{
+	"engine": "Redis",
+	"Redis": {
+		"host": "redis.example.com",
+		"port": 6379,
+		"username": "default",
+		"password": "your-redis-password"
+	}
+}
+```
+
+Redis `AUTH` is sent over the Redis connection itself, so use TLS too when traffic can cross an untrusted network, a shared network, or a managed provider endpoint that requires encryption.  The Redis engine passes extra config properties through to [ioredis](https://github.com/redis/ioredis), so you can enable TLS with the `tls` option:
+
+```json
+{
+	"engine": "Redis",
+	"Redis": {
+		"host": "redis.example.com",
+		"port": 6380,
+		"username": "default",
+		"password": "your-redis-password",
+		"tls": {}
+	}
+}
+```
+
+For a private CA or self-signed certificate, pass TLS options such as `ca`, `cert`, `key` or `servername` inside the `tls` object.  These are passed through to Node.js TLS.  If you store PEM text directly in JSON, preserve line breaks as `\n`:
+
+```json
+{
+	"engine": "Redis",
+	"Redis": {
+		"host": "redis.example.com",
+		"port": 6380,
+		"username": "default",
+		"password": "your-redis-password",
+		"tls": {
+			"ca": "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----\n"
+		}
+	}
+}
+```
+
+Please avoid `tls: { "rejectUnauthorized": false }` for production use.  It can be useful as a short diagnostic test, but it disables certificate verification.
+
 The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key-prefix) feature.  It allows you to prefix all the Redis keys with a common string, to separate your application's data in a shared database situation.
 
 The optional `keyTemplate` property works similarly to the [S3 Key Template](#s3-key-template) feature.  It allows you to specify an exact layout of MD5 hash characters, which can be prefixed, mixed in with or postfixed after the key.
@@ -648,6 +707,53 @@ Set the `host` and `port` for your own Redis cluster setup.  The `host` should p
 
 The `clusterOpts` property can hold several different cluster configuration options.  Please see [Cluster Options](https://redis.github.io/ioredis/interfaces/ClusterOptions.html) for other things you can include here, such as `scaleReads` and `redisOptions`.  For the `redisOptions` object, please see [Common Redis Options](https://redis.github.io/ioredis/interfaces/CommonRedisOptions.html) for other things you can include, such as timeouts, authentication and database selection.
 
+#### RedisCluster Auth and TLS
+
+For RedisCluster, shared authentication and TLS options should go inside `clusterOpts.redisOptions`, because those options are passed to every Redis node connection created by ioredis:
+
+```json
+{
+	"engine": "RedisCluster",
+	"RedisCluster": {
+		"host": "clustercfg.redis.example.com",
+		"port": 6379,
+		"clusterOpts": {
+			"scaleReads": "master",
+			"redisOptions": {
+				"commandTimeout": 5000,
+				"connectTimeout": 5000,
+				"username": "default",
+				"password": "your-cluster-password"
+			}
+		}
+	}
+}
+```
+
+To connect to a TLS-enabled Redis cluster, add `tls` under `redisOptions`:
+
+```json
+{
+	"engine": "RedisCluster",
+	"RedisCluster": {
+		"host": "clustercfg.redis.example.com",
+		"port": 6380,
+		"clusterOpts": {
+			"scaleReads": "master",
+			"redisOptions": {
+				"commandTimeout": 5000,
+				"connectTimeout": 5000,
+				"username": "default",
+				"password": "your-cluster-password",
+				"tls": {}
+			}
+		}
+	}
+}
+```
+
+AWS ElastiCache for Redis commonly uses this pattern when authentication and in-transit encryption are enabled.  Use `username` and `password` for ACL users.
+
 It is **highly recommended** that you keep the `scaleReads` property set to `"master"`, for immediate consistency (required for [Lists](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Lists.md), [Hashes](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Hashes.md), [Transactions](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Transactions.md) and the [Indexer](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Indexer.md)).
 
 The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key-prefix) feature.  It allows you to prefix all the Redis keys with a common string, to separate your application's data in a shared database situation.
@@ -752,7 +858,156 @@ You can customize the table name via the `table` configuration property.  It def
 
 [Connection Pooling](https://node-postgres.com/features/pooling) is utilized to open many simultaneous connections to greatly increase throughput.  Set the `max` property to the maximum number of concurrent connections your database can handle.  Ideally this should be set to the global [concurrency](#concurrency) setting.
 
-See [S3 Cache](#s3-cache) for details on the `cache` section, as it works in the same way.
+### Postgres SSL
+
+The Postgres engine passes all extra properties in the `Postgres` configuration block directly into the [pg.Pool constructor](https://node-postgres.com/apis/pool), except for `table`, `cache`, `passwordPlugin`, and `passwordPluginTimeout`, which are used by pixl-server-storage itself.  This means you can use the standard [node-postgres SSL options](https://node-postgres.com/features/ssl) without any special pixl-server-storage code.
+
+For a server with a certificate trusted by Node.js already, you can enable SSL by simply adding `"ssl":true` like this:
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"host": "postgres.example.com",
+		"database": "postgres",
+		"user": "postgres",
+		"password": "postgres",
+		"port": 5432,
+		"ssl": true
+	}
+}
+```
+
+For a private CA, self-signed server certificate, or hosted database that requires its own CA bundle, pass an `ssl` object.  This object is passed through to Node.js TLS, so you can provide `ca`, `cert`, `key`, `servername`, and other supported TLS options.  If you store PEM text directly in JSON, preserve line breaks as `\n` (standard JSON string encoding):
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"host": "postgres.example.com",
+		"database": "postgres",
+		"user": "postgres",
+		"password": "postgres",
+		"port": 5432,
+		"ssl": {
+			"ca": "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----\n"
+		}
+	}
+}
+```
+
+If you need to load the CA certificate from disk, you can also use a `connectionString` with the standard Postgres SSL parameters.  The `pg` module will read `sslrootcert` and use it as the TLS CA:
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"connectionString": "postgres://postgres:postgres@postgres.example.com:5432/postgres?sslmode=verify-full&sslrootcert=/path/to/root-ca.pem",
+		"max": 32,
+		"table": "items"
+	}
+}
+```
+
+If you put SSL parameters such as `sslmode`, `sslcert`, `sslkey`, or `sslrootcert` in the connection string, avoid also setting a separate `ssl` object.  The `pg` connection string parser replaces the `ssl` object when those URL parameters are present.
+
+Please avoid `ssl: { "rejectUnauthorized": false }` for production use.  It can be useful as a short diagnostic test, but it disables certificate verification.
+
+### AWS RDS
+
+Amazon RDS for PostgreSQL works well with this setup.  Download the appropriate RDS CA bundle from the [AWS RDS SSL/TLS documentation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html), place it somewhere readable by your app, and use `sslmode=verify-full` with `sslrootcert`:
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"connectionString": "postgres://xyops:secret@mydb.abc123.us-east-1.rds.amazonaws.com:5432/xyops?sslmode=verify-full&sslrootcert=/opt/xyops/conf/rds-ca.pem",
+		"max": 32,
+		"table": "items"
+	}
+}
+```
+
+AWS has a few quirks worth knowing:
+
+* RDS for PostgreSQL 15 and newer enables `rds.force_ssl` by default, so non-SSL connections may be rejected with an error similar to `no pg_hba.conf entry ... SSL off`.
+* `sslmode=verify-full` verifies both the certificate chain and the hostname.  Use the actual RDS endpoint hostname in your connection string when possible.
+* If you connect through a custom DNS name, SSH tunnel, or local proxy, hostname verification may need the original RDS endpoint as the TLS `servername` option.  In that case, prefer explicit config properties and an `ssl` object instead of URL SSL parameters.
+* AWS recommends trusting the root RDS CA certificate.  Avoid pinning intermediate certificates, as that can cause trouble when RDS rotates server certificates.
+
+### Postgres Password Plugin
+
+If your Postgres password needs to be generated dynamically, such as with a cloud provider token service, you can configure a password plugin command.  This uses the [node-postgres dynamic password callback](https://node-postgres.com/features/connecting), so the command is called whenever the pool needs a new physical database connection.
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"host": "mypg.postgres.example.com",
+		"database": "mydb",
+		"user": "my-app-user",
+		"ssl": true,
+		"passwordPlugin": "npx -y pxpg-plug-example@1.0.0"
+	}
+}
+```
+
+In this example the command uses the [npx](https://docs.npmjs.com/cli/commands/npx) CLI to automatically download and run a NPM package.  The package is also cached on local disk (in `~/.npm`) for fast subsequent runs.
+
+The command receives one compact JSON document on STDIN, delimited by an EOL (pretty-printed here for display purposes):
+
+```json
+{
+	"type": "postgres_password",
+	"config": {
+		"host": "mypg.postgres.example.com",
+		"database": "mydb",
+		"user": "my-app-user",
+		"ssl": true
+	}
+}
+```
+
+The `config` object contains the Postgres configuration with `password`, `passwordPlugin`, and `passwordPluginTimeout` removed.
+
+The command should print one compact JSON document to STDOUT.  Extra logging is allowed, as pixl-server-storage only parses the last line of STDOUT:
+
+```json
+{
+	"password": "GENERATED_PASSWORD_OR_TOKEN",
+	"ttl": 3000
+}
+```
+
+The `password` property is required, and must be a string.  The optional `ttl` property is a cache duration in seconds.  If `ttl` is greater than zero, pixl-server-storage caches the returned password in memory and reuses it for subsequent new database connections until the TTL expires.  If `ttl` is omitted or zero, the command is invoked for every new database connection.
+
+If the plugin cannot produce a password, it can return an error:
+
+```json
+{
+	"code": "auth",
+	"description": "Could not fetch provider token."
+}
+```
+
+The default plugin timeout is 30 seconds.  You can customize it with `passwordPluginTimeout`, in milliseconds:
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"host": "mypg.postgres.example.com",
+		"database": "mydb",
+		"user": "my-app-user",
+		"passwordPlugin": "/usr/local/bin/px-pg-plug-example",
+		"passwordPluginTimeout": 10000
+	}
+}
+```
+
+For production, consider preinstalling the plugin and pointing `passwordPlugin` at the installed executable.  This avoids starting `npx` and checking the NPM registry whenever the cached password expires.
+
+See [S3 Cache](#s3-cache) for details on the `cache` section, as Postgres works in the same way.
 
 Note that pooling and caching have little effect when the database is running locally (i.e. localhost).  They offer much more benefit when the database is located remotely over some number of network hops.
 

package/docs/API.md

@@ -24,6 +24,7 @@ let storage = server.Storage;
 	* [headMulti](#headmulti)
 	* [delete](#delete)
 	* [deleteMulti](#deletemulti)
+	* [deleteAll](#deleteall)
 	* [copy](#copy)
 	* [rename](#rename)
 	* [lock](#lock)
@@ -328,6 +329,22 @@ storage.deleteMulti( ['test1', 'test2', 'test3'], function(err) {
 } );
 ```
 
+Note that this function returns immediately if it fails to delete any key in the specified set.
+
+## deleteAll
+
+```js
+storage.deleteAll( KEYS, CALLBACK );
+```
+
+The `deleteAll()` method deletes multiple records at once, from a specified array of keys.  This is similar to [deleteMulti()](#deletemulti), except it does not stop on the first error.  Each delete error is logged, and the remaining keys are still attempted.  Example:
+
+```js
+storage.deleteAll( ['test1', 'test2', 'test3'], function(err) {
+	if (err) throw err;
+} );
+```
+
 ## copy
 
 ```js

package/engines/Postgres.js

@@ -7,6 +7,8 @@
 const fs = require('fs');
 const Path = require('path');
 const zlib = require('zlib');
+const os = require('os');
+const cp = require('child_process');
 const Class = require("pixl-class");
 const Component = require("pixl-server/component");
 const pg = require('pg');
@@ -33,6 +35,8 @@ module.exports = Class.create({
 		query_timeout: 6000,
 		connectionTimeoutMillis: 30000,
 		idleTimeoutMillis: 10000,
+		passwordPlugin: "",
+		passwordPluginTimeout: 30000,
 		table: "items"
 	},
 	
@@ -75,9 +79,16 @@ module.exports = Class.create({
 		}
 		
 		// pass entire config to pg.Pool, sans our custom keys
-		var db = this.db = new pg.Pool( 
-			Tools.copyHashRemoveKeys( pg_config, { cache: 1, table: 1 }) 
-		);
+		var pool_config = Tools.copyHashRemoveKeys( pg_config, { cache: 1, table: 1, passwordPlugin: 1, passwordPluginTimeout: 1 } );
+		
+		if (pg_config.passwordPlugin) {
+			// node-postgres supports password as a sync/async callback, called for each
+			// new physical connection.  We use that hook to ask an external command for
+			// short-lived provider tokens without baking provider-specific code in here.
+			pool_config.password = this.getPasswordFromPlugin.bind(this);
+		}
+		
+		var db = this.db = new pg.Pool( pool_config );
 		
 		db.on('error', function(err) {
 			self.logError('db', "DB Error: " + err, err);
@@ -104,6 +115,89 @@ module.exports = Class.create({
 		this.queryWithClient( this.db, sql, args, callback );
 	},
 	
+	getPasswordFromPlugin: function() {
+		// execute an external command to resolve the Postgres password
+		// the plugin may also return a TTL, allowing us to cache it in memory
+		var self = this;
+		var pg_config = this.config.get();
+		var now = Date.now();
+		
+		if (this.passwordPluginCache && (now < this.passwordPluginCache.expires)) {
+			return Promise.resolve( this.passwordPluginCache.password );
+		}
+		
+		// If a plugin call is already in-flight, return the same Promise.
+		// This coalesces simultaneous pool reconnects into one plugin exec.
+		if (this.passwordPluginFetch) return this.passwordPluginFetch;
+		
+		var hook_args = {
+			type: 'postgres_password',
+			config: Tools.copyHashRemoveKeys( pg_config, { password: 1, passwordPlugin: 1, passwordPluginTimeout: 1 } )
+		};
+		
+		var child_cmd = pg_config.passwordPlugin;
+		var child_opts = {
+			cwd: os.tmpdir(),
+			timeout: pg_config.passwordPluginTimeout || 30000
+		};
+		
+		this.logDebug(5, "Calling Postgres password plugin");
+		
+		this.passwordPluginFetch = new Promise( function(resolve, reject) {
+			var child = cp.exec( child_cmd, child_opts, function(err, stdout, stderr) {
+				var json = null;
+				
+				if (!err && stdout.match(/\S/)) {
+					// parse last line only, to omit any noise from plugin
+					try { json = JSON.parse( stdout.replace(/\r\n/g, "\n").trim().split(/\n/).pop() ); }
+					catch (e) {
+						err = new Error("Postgres password plugin JSON parse error: " + (e.message || e));
+						err.code = 'json';
+					}
+				}
+				
+				if (stderr && stderr.match(/\S/)) {
+					self.logDebug(9, "Postgres password plugin emitted STDERR: " + Buffer.byteLength(stderr) + " bytes");
+				}
+				
+				if (err) return reject( err );
+				if (!json) return reject( new Error("Postgres password plugin error: No JSON found in response STDOUT") );
+				if (json.code) return reject( new Error("Postgres password plugin error: " + (json.description || json.code)) );
+				if (typeof(json.password) != 'string') return reject( new Error("Postgres password plugin error: No password string found in response JSON") );
+				
+				var ttl = parseInt( json.ttl || 0, 10 ) || 0;
+				
+				if (ttl > 0) {
+					self.passwordPluginCache = {
+						password: json.password,
+						expires: Date.now() + (ttl * 1000)
+					};
+					
+					self.logDebug(6, "Postgres password plugin returned password with TTL: " + ttl + " seconds");
+				}
+				else {
+					self.passwordPluginCache = null;
+					self.logDebug(6, "Postgres password plugin returned password without TTL");
+				}
+				
+				resolve( json.password );
+			} ); // cp.exec
+			
+			// Write hook data to child's stdin, using one compact JSON document.
+			child.stdin.on('error', noop);
+			child.stdin.write( JSON.stringify(hook_args) + "\n" );
+			child.stdin.end();
+		} ).then( function(password) {
+			self.passwordPluginFetch = null;
+			return password;
+		}, function(err) {
+			self.passwordPluginFetch = null;
+			throw err;
+		} );
+		
+		return this.passwordPluginFetch;
+	},
+	
 	queryWithClient: function(client, sql, args, callback) {
 		// run and log db query, and error if applicable
 		// allow for named variables in sql, expand to numbered $ variables for pg

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "4.2.1",
+	"version": "4.2.2",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",

package/storage.js

@@ -516,6 +516,35 @@ module.exports = Class.create({
 		);
 	},
 	
+	deleteAll: function(keys, callback) {
+		// delete multiple records at once, given array of keys
+		// continue through all keys, even if some deletes fail
+		var self = this;
+		if (!this.started) return callback( new Error("Storage has not completed startup.") );
+		
+		// if engine provides its own deleteAll, call that directly
+		if (("deleteAll" in this.engine) && !this.currentTransactionPath) {
+			return this.engine.deleteAll(keys, callback);
+		}
+		
+		async.eachLimit(keys, this.concurrency, 
+			function(key, callback) {
+				// iterator for each key
+				self.delete(key, function(err) {
+					if (err) {
+						// log and swallow the error so the remaining deletes can proceed
+						self.logError('storage', "Failed to delete key in deleteAll: " + key + ": " + err);
+					}
+					callback();
+				} );
+			}, 
+			function() {
+				// all keys attempted
+				callback();
+			}
+		);
+	},
+	
 	copy: function(old_key, new_key, callback) {
 		// copy record to new key
 		var self = this;

package/test/test-main.js

@@ -656,6 +656,38 @@ module.exports = {
 			} );
 		},
 		
+		function testDeleteAll(test) {
+			// delete multiple keys using deleteAll, continuing past errors
+			test.expect(5);
+			var self = this;
+			var keys = ['deleteall1', 'deleteall_missing', 'deleteall2'];
+			
+			this.storage.putMulti( {
+				deleteall1: { fruit: 'apple' },
+				deleteall2: { fruit: 'banana' }
+			}, function(err) {
+				test.ok( !err, "No error creating deleteAll test records: " + err );
+				
+				self.storage.deleteAll( keys, function(err) {
+					test.ok( !err, "No error calling deleteAll: " + err );
+					
+					// the missing middle key should not prevent the final key from deleting
+					self.storage.get( 'deleteall1', function(err) {
+						test.ok( !!err, "Expected error fetching first deleteAll key after delete" );
+						
+						self.storage.get( 'deleteall_missing', function(err) {
+							test.ok( !!err, "Expected error fetching missing deleteAll key after delete" );
+							
+							self.storage.get( 'deleteall2', function(err) {
+								test.ok( !!err, "Expected error fetching final deleteAll key after delete" );
+								test.done();
+							} );
+						} );
+					} );
+				} );
+			} );
+		},
+		
 		function testOptimize(test) {
 			// run engine-specific optimization, or a safe no-op if unsupported
 			test.expect(8);

package/test/test-postgres-password-plugin.js

@@ -0,0 +1,243 @@
+// Unit tests for Postgres password plugin support
+// Copyright (c) 2026 Joseph Huckaby
+// Released under the MIT License
+
+var cp = require('child_process');
+var EventEmitter = require('events').EventEmitter;
+var Module = require('module');
+
+var origLoad = Module._load;
+var origExec = cp.exec;
+
+// The Postgres engine requires 'pg' at module load time.  Stub it here so this
+// focused test can run without needing a live Postgres module or database.
+var fakePG = {
+	lastConfig: null,
+	Pool: function(config) {
+		fakePG.lastConfig = config;
+		this.on = function() {};
+		this.query = function(sql, args, callback) {
+			callback( null, { command: 'CREATE', rowCount: 0, rows: [] } );
+		};
+	}
+};
+
+Module._load = function(request, parent, isMain) {
+	if (request == 'pg') return fakePG;
+	return origLoad.apply( this, arguments );
+};
+
+var PostgresEngine = require('../engines/Postgres.js');
+Module._load = origLoad;
+
+function makeEngine(config) {
+	// Keep this intentionally tiny.  The password plugin helper only needs the
+	// component config and log methods, and setup() only needs query logging.
+	var engine = Object.create( PostgresEngine.prototype );
+	engine.config = {
+		get: function() { return config; }
+	};
+	engine.logDebug = function() {};
+	engine.logError = function() {};
+	return engine;
+}
+
+function mockExec(responder) {
+	var calls = [];
+	
+	cp.exec = function(cmd, opts, callback) {
+		var call = { cmd: cmd, opts: opts, stdin: '' };
+		var child = { stdin: new EventEmitter() };
+		calls.push( call );
+		
+		child.stdin.write = function(data) {
+			call.stdin += data;
+		};
+		child.stdin.end = function() {
+			responder( call, callback );
+		};
+		
+		return child;
+	};
+	
+	return calls;
+}
+
+function restoreExec() {
+	cp.exec = origExec;
+}
+
+module.exports = {
+	tests: [
+		
+		function setup_installsPasswordFunctionAndStripsPluginKeys(test) {
+			test.expect(4);
+			
+			var engine = makeEngine({
+				host: 'localhost',
+				database: 'test',
+				user: 'test',
+				password: 'static',
+				passwordPlugin: 'node ./plugin.js',
+				passwordPluginTimeout: 1234,
+				table: 'items'
+			});
+			
+			fakePG.lastConfig = null;
+			engine.setup( function(err) {
+				test.ok( !err, "Setup completed without error" );
+				test.ok( typeof(fakePG.lastConfig.password) == 'function', "Password was converted to a callback function" );
+				test.ok( !('passwordPlugin' in fakePG.lastConfig), "passwordPlugin was stripped from pool config" );
+				test.ok( !('passwordPluginTimeout' in fakePG.lastConfig), "passwordPluginTimeout was stripped from pool config" );
+				test.done();
+			} );
+		},
+		
+		function passwordPlugin_returnsPasswordAndSendsConfig(test) {
+			test.expect(5);
+			
+			var engine = makeEngine({
+				host: 'localhost',
+				database: 'test',
+				user: 'tester',
+				password: 'static',
+				passwordPlugin: 'node ./plugin.js',
+				passwordPluginTimeout: 1234,
+				table: 'items'
+			});
+			
+			var calls = mockExec( function(call, callback) {
+				callback( null, JSON.stringify({ xy: 1, password: 'dynamic-secret' }) + "\n", "" );
+			} );
+			
+			engine.getPasswordFromPlugin().then( function(password) {
+				var input = JSON.parse( calls[0].stdin );
+				
+				test.ok( password == 'dynamic-secret', "Returned dynamic password" );
+				test.ok( calls[0].cmd == 'node ./plugin.js', "Used configured plugin command" );
+				test.ok( calls[0].opts.timeout == 1234, "Used configured plugin timeout" );
+				test.ok( input.type == 'postgres_password', "Sent expected hook type" );
+				test.ok( !('password' in input.config), "Static password was omitted from plugin input" );
+				
+				restoreExec();
+				test.done();
+			} ).catch( function(err) {
+				restoreExec();
+				test.ok( false, "Unexpected error: " + err );
+				test.done();
+			} );
+		},
+		
+		function passwordPlugin_cachesPasswordForTTL(test) {
+			test.expect(3);
+			
+			var engine = makeEngine({
+				passwordPlugin: 'node ./plugin.js',
+				table: 'items'
+			});
+			
+			var calls = mockExec( function(call, callback) {
+				callback( null, JSON.stringify({ xy: 1, password: 'cached-secret', ttl: 60 }) + "\n", "" );
+			} );
+			
+			engine.getPasswordFromPlugin().then( function(first) {
+				return engine.getPasswordFromPlugin().then( function(second) {
+					test.ok( first == 'cached-secret', "First password was returned" );
+					test.ok( second == 'cached-secret', "Second password was returned from cache" );
+					test.ok( calls.length == 1, "Plugin was only invoked once" );
+					
+					restoreExec();
+					test.done();
+				} );
+			} ).catch( function(err) {
+				restoreExec();
+				test.ok( false, "Unexpected error: " + err );
+				test.done();
+			} );
+		},
+		
+		function passwordPlugin_doesNotCacheWithoutTTL(test) {
+			test.expect(2);
+			
+			var counter = 0;
+			var engine = makeEngine({
+				passwordPlugin: 'node ./plugin.js',
+				table: 'items'
+			});
+			
+			var calls = mockExec( function(call, callback) {
+				counter++;
+				callback( null, JSON.stringify({ xy: 1, password: 'secret-' + counter }) + "\n", "" );
+			} );
+			
+			engine.getPasswordFromPlugin().then( function(first) {
+				return engine.getPasswordFromPlugin().then( function(second) {
+					test.ok( first != second, "Passwords were fetched separately" );
+					test.ok( calls.length == 2, "Plugin was invoked twice" );
+					
+					restoreExec();
+					test.done();
+				} );
+			} ).catch( function(err) {
+				restoreExec();
+				test.ok( false, "Unexpected error: " + err );
+				test.done();
+			} );
+		},
+		
+		function passwordPlugin_coalescesConcurrentFetches(test) {
+			test.expect(2);
+			
+			var engine = makeEngine({
+				passwordPlugin: 'node ./plugin.js',
+				table: 'items'
+			});
+			
+			var calls = mockExec( function(call, callback) {
+				setTimeout( function() {
+					callback( null, JSON.stringify({ xy: 1, password: 'shared-secret', ttl: 60 }) + "\n", "" );
+				}, 10 );
+			} );
+			
+			Promise.all([
+				engine.getPasswordFromPlugin(),
+				engine.getPasswordFromPlugin(),
+				engine.getPasswordFromPlugin()
+			]).then( function(passwords) {
+				test.ok( passwords.join(',') == 'shared-secret,shared-secret,shared-secret', "All callers received the same password" );
+				test.ok( calls.length == 1, "Only one plugin process was spawned" );
+				
+				restoreExec();
+				test.done();
+			} ).catch( function(err) {
+				restoreExec();
+				test.ok( false, "Unexpected error: " + err );
+				test.done();
+			} );
+		},
+		
+		function passwordPlugin_reportsPluginError(test) {
+			test.expect(1);
+			
+			var engine = makeEngine({
+				passwordPlugin: 'node ./plugin.js',
+				table: 'items'
+			});
+			
+			mockExec( function(call, callback) {
+				callback( null, JSON.stringify({ xy: 1, code: 'auth', description: 'Token rejected' }) + "\n", "" );
+			} );
+			
+			engine.getPasswordFromPlugin().then( function() {
+				restoreExec();
+				test.ok( false, "Expected plugin error" );
+				test.done();
+			} ).catch( function(err) {
+				restoreExec();
+				test.ok( /Token rejected/.test(err.message), "Reported plugin error: " + err.message );
+				test.done();
+			} );
+		}
+		
+	]
+};