pixl-server-storage 4.2.1 → 4.3.0

package/README.md

@@ -56,10 +56,16 @@ Here is the table of contents for this current document:
 		+ [S3 Cache](#s3-cache)
 		+ [S3 Compatible Services](#s3-compatible-services)
 	* [Redis](#redis)
+		+ [Redis Auth and TLS](#redis-auth-and-tls)
 		+ [RedisCluster](#rediscluster)
+			- [RedisCluster Auth and TLS](#rediscluster-auth-and-tls)
 	* [SQLite](#sqlite)
 	* [Postgres](#postgres)
+		+ [Postgres SSL](#postgres-ssl)
+		+ [AWS RDS](#aws-rds)
+		+ [Postgres Password Plugin](#postgres-password-plugin)
 	* [Hybrid](#hybrid)
+- [Cache Buffer Copying](#cache-buffer-copying)
 - [Key Normalization](#key-normalization)
 - [Basic Functions](#basic-functions)
 	* [Storing Records](#storing-records)
@@ -412,8 +418,9 @@ The properties are as follows:
 | `enabled` | Boolean | Set this to `true` to enable the filesystem caching system. |
 | `maxItems` | Integer | This is the maximum number of objects to allow in the cache. |
 | `maxBytes` | Integer | This is the maximum number of bytes to allow in the cache. |
+| `copyBuffers` | Boolean | Defaults to `true`.  Stores independent, unpooled copies of cached Buffers.  Set to `false` to opt out.  See [Cache Buffer Copying](#cache-buffer-copying). |
 
-The cache will automatically expire objects in LRU fashion when either of the limits are exceeded (whichever is hit first).  Set the properties to `0` for no limit.
+The cache will automatically expire objects in LRU fashion when either of the limits are exceeded (whichever is hit first).  Set either limit to `0` for no limit.
 
 Note that binary records are **not** cached.  This system is for JSON records only.
 
@@ -513,8 +520,9 @@ The properties are as follows:
 | `enabled` | Boolean | Set this to `true` to enable the S3 caching system. |
 | `maxItems` | Integer | This is the maximum number of objects to allow in the cache. |
 | `maxBytes` | Integer | This is the maximum number of bytes to allow in the cache. |
+| `copyBuffers` | Boolean | Defaults to `true`.  Stores independent, unpooled copies of cached Buffers.  Set to `false` to opt out.  See [Cache Buffer Copying](#cache-buffer-copying). |
 
-The cache will automatically expire objects in LRU fashion when either of the limits are met (whichever is reached first).  Set the properties to `0` for no limit.
+The cache will automatically expire objects in LRU fashion when either of the limits are met (whichever is reached first).  Set either limit to `0` for no limit.
 
 It is recommended that you set the `maxItems` and `maxBytes` high enough to allow new data written to live for *at least* several seconds before getting expired out of the cache.  This depends on the overall storage throughput of your application, but 1,000 max items and 10 MB max bytes is probably fine for most use cases.
 
@@ -590,6 +598,8 @@ Then configure your storage thusly:
 	"Redis": {
 		"host": "127.0.0.1",
 		"port": 6379,
+		"username": "",
+		"password": "",
 		"keyPrefix": "",
 		"keyTemplate": "",
 		"cache": {
@@ -603,6 +613,58 @@ Then configure your storage thusly:
 
 Set the `host` and `port` for your own Redis server setup.  Please see [Common Redis Options](https://redis.github.io/ioredis/interfaces/CommonRedisOptions.html) for other things you can include here, such as timeouts, authentication and database selection.
 
+### Redis Auth and TLS
+
+For production, Redis is usually deployed on a private network, firewalled to trusted clients, and protected with authentication as an extra layer of safety.  Modern Redis ACL users use both `username` and `password`.
+
+Configure `username` and `password` like this:
+
+```json
+{
+	"engine": "Redis",
+	"Redis": {
+		"host": "redis.example.com",
+		"port": 6379,
+		"username": "default",
+		"password": "your-redis-password"
+	}
+}
+```
+
+Redis `AUTH` is sent over the Redis connection itself, so use TLS too when traffic can cross an untrusted network, a shared network, or a managed provider endpoint that requires encryption.  The Redis engine passes extra config properties through to [ioredis](https://github.com/redis/ioredis), so you can enable TLS with the `tls` option:
+
+```json
+{
+	"engine": "Redis",
+	"Redis": {
+		"host": "redis.example.com",
+		"port": 6380,
+		"username": "default",
+		"password": "your-redis-password",
+		"tls": {}
+	}
+}
+```
+
+For a private CA or self-signed certificate, pass TLS options such as `ca`, `cert`, `key` or `servername` inside the `tls` object.  These are passed through to Node.js TLS.  If you store PEM text directly in JSON, preserve line breaks as `\n`:
+
+```json
+{
+	"engine": "Redis",
+	"Redis": {
+		"host": "redis.example.com",
+		"port": 6380,
+		"username": "default",
+		"password": "your-redis-password",
+		"tls": {
+			"ca": "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----\n"
+		}
+	}
+}
+```
+
+Please avoid `tls: { "rejectUnauthorized": false }` for production use.  It can be useful as a short diagnostic test, but it disables certificate verification.
+
 The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key-prefix) feature.  It allows you to prefix all the Redis keys with a common string, to separate your application's data in a shared database situation.
 
 The optional `keyTemplate` property works similarly to the [S3 Key Template](#s3-key-template) feature.  It allows you to specify an exact layout of MD5 hash characters, which can be prefixed, mixed in with or postfixed after the key.
@@ -648,6 +710,53 @@ Set the `host` and `port` for your own Redis cluster setup.  The `host` should p
 
 The `clusterOpts` property can hold several different cluster configuration options.  Please see [Cluster Options](https://redis.github.io/ioredis/interfaces/ClusterOptions.html) for other things you can include here, such as `scaleReads` and `redisOptions`.  For the `redisOptions` object, please see [Common Redis Options](https://redis.github.io/ioredis/interfaces/CommonRedisOptions.html) for other things you can include, such as timeouts, authentication and database selection.
 
+#### RedisCluster Auth and TLS
+
+For RedisCluster, shared authentication and TLS options should go inside `clusterOpts.redisOptions`, because those options are passed to every Redis node connection created by ioredis:
+
+```json
+{
+	"engine": "RedisCluster",
+	"RedisCluster": {
+		"host": "clustercfg.redis.example.com",
+		"port": 6379,
+		"clusterOpts": {
+			"scaleReads": "master",
+			"redisOptions": {
+				"commandTimeout": 5000,
+				"connectTimeout": 5000,
+				"username": "default",
+				"password": "your-cluster-password"
+			}
+		}
+	}
+}
+```
+
+To connect to a TLS-enabled Redis cluster, add `tls` under `redisOptions`:
+
+```json
+{
+	"engine": "RedisCluster",
+	"RedisCluster": {
+		"host": "clustercfg.redis.example.com",
+		"port": 6380,
+		"clusterOpts": {
+			"scaleReads": "master",
+			"redisOptions": {
+				"commandTimeout": 5000,
+				"connectTimeout": 5000,
+				"username": "default",
+				"password": "your-cluster-password",
+				"tls": {}
+			}
+		}
+	}
+}
+```
+
+AWS ElastiCache for Redis commonly uses this pattern when authentication and in-transit encryption are enabled.  Use `username` and `password` for ACL users.
+
 It is **highly recommended** that you keep the `scaleReads` property set to `"master"`, for immediate consistency (required for [Lists](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Lists.md), [Hashes](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Hashes.md), [Transactions](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Transactions.md) and the [Indexer](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Indexer.md)).
 
 The optional `keyPrefix` property works similarly to the [S3 Key Prefix](#s3-key-prefix) feature.  It allows you to prefix all the Redis keys with a common string, to separate your application's data in a shared database situation.
@@ -752,7 +861,156 @@ You can customize the table name via the `table` configuration property.  It def
 
 [Connection Pooling](https://node-postgres.com/features/pooling) is utilized to open many simultaneous connections to greatly increase throughput.  Set the `max` property to the maximum number of concurrent connections your database can handle.  Ideally this should be set to the global [concurrency](#concurrency) setting.
 
-See [S3 Cache](#s3-cache) for details on the `cache` section, as it works in the same way.
+### Postgres SSL
+
+The Postgres engine passes all extra properties in the `Postgres` configuration block directly into the [pg.Pool constructor](https://node-postgres.com/apis/pool), except for `table`, `cache`, `passwordPlugin`, and `passwordPluginTimeout`, which are used by pixl-server-storage itself.  This means you can use the standard [node-postgres SSL options](https://node-postgres.com/features/ssl) without any special pixl-server-storage code.
+
+For a server with a certificate trusted by Node.js already, you can enable SSL by simply adding `"ssl":true` like this:
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"host": "postgres.example.com",
+		"database": "postgres",
+		"user": "postgres",
+		"password": "postgres",
+		"port": 5432,
+		"ssl": true
+	}
+}
+```
+
+For a private CA, self-signed server certificate, or hosted database that requires its own CA bundle, pass an `ssl` object.  This object is passed through to Node.js TLS, so you can provide `ca`, `cert`, `key`, `servername`, and other supported TLS options.  If you store PEM text directly in JSON, preserve line breaks as `\n` (standard JSON string encoding):
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"host": "postgres.example.com",
+		"database": "postgres",
+		"user": "postgres",
+		"password": "postgres",
+		"port": 5432,
+		"ssl": {
+			"ca": "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----\n"
+		}
+	}
+}
+```
+
+If you need to load the CA certificate from disk, you can also use a `connectionString` with the standard Postgres SSL parameters.  The `pg` module will read `sslrootcert` and use it as the TLS CA:
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"connectionString": "postgres://postgres:postgres@postgres.example.com:5432/postgres?sslmode=verify-full&sslrootcert=/path/to/root-ca.pem",
+		"max": 32,
+		"table": "items"
+	}
+}
+```
+
+If you put SSL parameters such as `sslmode`, `sslcert`, `sslkey`, or `sslrootcert` in the connection string, avoid also setting a separate `ssl` object.  The `pg` connection string parser replaces the `ssl` object when those URL parameters are present.
+
+Please avoid `ssl: { "rejectUnauthorized": false }` for production use.  It can be useful as a short diagnostic test, but it disables certificate verification.
+
+### AWS RDS
+
+Amazon RDS for PostgreSQL works well with this setup.  Download the appropriate RDS CA bundle from the [AWS RDS SSL/TLS documentation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html), place it somewhere readable by your app, and use `sslmode=verify-full` with `sslrootcert`:
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"connectionString": "postgres://xyops:secret@mydb.abc123.us-east-1.rds.amazonaws.com:5432/xyops?sslmode=verify-full&sslrootcert=/opt/xyops/conf/rds-ca.pem",
+		"max": 32,
+		"table": "items"
+	}
+}
+```
+
+AWS has a few quirks worth knowing:
+
+* RDS for PostgreSQL 15 and newer enables `rds.force_ssl` by default, so non-SSL connections may be rejected with an error similar to `no pg_hba.conf entry ... SSL off`.
+* `sslmode=verify-full` verifies both the certificate chain and the hostname.  Use the actual RDS endpoint hostname in your connection string when possible.
+* If you connect through a custom DNS name, SSH tunnel, or local proxy, hostname verification may need the original RDS endpoint as the TLS `servername` option.  In that case, prefer explicit config properties and an `ssl` object instead of URL SSL parameters.
+* AWS recommends trusting the root RDS CA certificate.  Avoid pinning intermediate certificates, as that can cause trouble when RDS rotates server certificates.
+
+### Postgres Password Plugin
+
+If your Postgres password needs to be generated dynamically, such as with a cloud provider token service, you can configure a password plugin command.  This uses the [node-postgres dynamic password callback](https://node-postgres.com/features/connecting), so the command is called whenever the pool needs a new physical database connection.
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"host": "mypg.postgres.example.com",
+		"database": "mydb",
+		"user": "my-app-user",
+		"ssl": true,
+		"passwordPlugin": "npx -y pxpg-plug-example@1.0.0"
+	}
+}
+```
+
+In this example the command uses the [npx](https://docs.npmjs.com/cli/commands/npx) CLI to automatically download and run a NPM package.  The package is also cached on local disk (in `~/.npm`) for fast subsequent runs.
+
+The command receives one compact JSON document on STDIN, delimited by an EOL (pretty-printed here for display purposes):
+
+```json
+{
+	"type": "postgres_password",
+	"config": {
+		"host": "mypg.postgres.example.com",
+		"database": "mydb",
+		"user": "my-app-user",
+		"ssl": true
+	}
+}
+```
+
+The `config` object contains the Postgres configuration with `password`, `passwordPlugin`, and `passwordPluginTimeout` removed.
+
+The command should print one compact JSON document to STDOUT.  Extra logging is allowed, as pixl-server-storage only parses the last line of STDOUT:
+
+```json
+{
+	"password": "GENERATED_PASSWORD_OR_TOKEN",
+	"ttl": 3000
+}
+```
+
+The `password` property is required, and must be a string.  The optional `ttl` property is a cache duration in seconds.  If `ttl` is greater than zero, pixl-server-storage caches the returned password in memory and reuses it for subsequent new database connections until the TTL expires.  If `ttl` is omitted or zero, the command is invoked for every new database connection.
+
+If the plugin cannot produce a password, it can return an error:
+
+```json
+{
+	"code": "auth",
+	"description": "Could not fetch provider token."
+}
+```
+
+The default plugin timeout is 30 seconds.  You can customize it with `passwordPluginTimeout`, in milliseconds:
+
+```json
+{
+	"engine": "Postgres",
+	"Postgres": {
+		"host": "mypg.postgres.example.com",
+		"database": "mydb",
+		"user": "my-app-user",
+		"passwordPlugin": "/usr/local/bin/px-pg-plug-example",
+		"passwordPluginTimeout": 10000
+	}
+}
+```
+
+For production, consider preinstalling the plugin and pointing `passwordPlugin` at the installed executable.  This avoids starting `npx` and checking the NPM registry whenever the cached password expires.
+
+See [S3 Cache](#s3-cache) for details on the `cache` section, as Postgres works in the same way.
 
 Note that pooling and caching have little effect when the database is running locally (i.e. localhost).  They offer much more benefit when the database is located remotely over some number of network hops.
 
@@ -800,6 +1058,31 @@ Note that all of the engine configuration objects are on the same level as the `
 
 When transactions are enabled, the `Hybrid` engine routes transaction commits through the configured `docEngine`, because pixl-server-storage transactions only apply to JSON records.  If the `docEngine` supports native transactions, such as Postgres, Redis or SQLite, `Hybrid` will use that native commit path automatically.  The `binaryEngine` is not involved in transactions, because binary keys are passed straight through.
 
+# Cache Buffer Copying
+
+All six engine caches (`Filesystem`, `S3`, `Redis`, `RedisCluster`, `SQLite`, and `Postgres`) default to `copyBuffers: true`.  This applies whenever the engine's cache is enabled with `cache.enabled: true`; existing cache configurations receive the new default automatically.
+
+These caches store serialized JSON records as Buffers.  A small Buffer can share a much larger memory allocation with other Buffers, and keeping it in the cache can retain that entire allocation.  Node.js 24.18.0 increased the default Buffer pool size from 8 KiB to 64 KiB, making this retention more costly.  See the [Node.js 24.18.0 release notes](https://nodejs.org/en/blog/release/v24.18.0).
+
+The Postgres report in [issue #8](https://github.com/jhuckaby/pixl-server-storage/issues/8) demonstrated how retained backing allocations could greatly exceed the cached payload size.  The same retention pattern can affect all six engines, so Buffer copying is enabled by default across their caches.  Each cached Buffer gets its own unpooled copy when added or replaced, preventing it from retaining an oversized shared allocation.  This adds an allocation and byte copy on insertion, but cache hits do not make additional Buffer copies.  The `maxBytes` setting still measures cache key and value sizes, rather than total process memory.
+
+If you need to avoid the extra copying, set `copyBuffers` to `false` inside your engine's `cache` configuration.  For example, merge this fragment into your existing `Storage` configuration for Postgres:
+
+```json
+{
+	"Postgres": {
+		"cache": {
+			"enabled": true,
+			"maxItems": 100000,
+			"maxBytes": 104857600,
+			"copyBuffers": false
+		}
+	}
+}
+```
+
+Use the same `cache.copyBuffers` property inside any of the other five engine configuration blocks.  Caching remains enabled in this example, while disabling Buffer copying restores the previous retention behavior.  Omit `copyBuffers`, or set it to `true`, to use the recommended default.
+
 # Key Normalization
 
 In order to maintain compatibility with all the various engines, keys are "normalized" on all entry points.  Specifically, they undergo the following transformations before being passed along to the engine:

package/docs/API.md

@@ -24,6 +24,7 @@ let storage = server.Storage;
 	* [headMulti](#headmulti)
 	* [delete](#delete)
 	* [deleteMulti](#deletemulti)
+	* [deleteAll](#deleteall)
 	* [copy](#copy)
 	* [rename](#rename)
 	* [lock](#lock)
@@ -328,6 +329,22 @@ storage.deleteMulti( ['test1', 'test2', 'test3'], function(err) {
 } );
 ```
 
+Note that this function returns immediately if it fails to delete any key in the specified set.
+
+## deleteAll
+
+```js
+storage.deleteAll( KEYS, CALLBACK );
+```
+
+The `deleteAll()` method deletes multiple records at once, from a specified array of keys.  This is similar to [deleteMulti()](#deletemulti), except it does not stop on the first error.  Each delete error is logged, and the remaining keys are still attempted.  Example:
+
+```js
+storage.deleteAll( ['test1', 'test2', 'test3'], function(err) {
+	if (err) throw err;
+} );
+```
+
 ## copy
 
 ```js

package/engines/Filesystem.js

@@ -80,7 +80,7 @@ module.exports = Class.create({
 		var cache_opts = this.config.get('cache');
 		if (cache_opts && cache_opts.enabled) {
 			this.logDebug(3, "Setting up LRU cache", cache_opts);
-			this.cache = new Cache( Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) );
+			this.cache = new Cache({ copyBuffers: true, ...Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) });
 			this.cache.on('expire', function(item, reason) {
 				self.logDebug(9, "Expiring LRU cache object: " + item.key + " due to: " + reason, {
 					key: item.key,

package/engines/Postgres.js

@@ -7,6 +7,8 @@
 const fs = require('fs');
 const Path = require('path');
 const zlib = require('zlib');
+const os = require('os');
+const cp = require('child_process');
 const Class = require("pixl-class");
 const Component = require("pixl-server/component");
 const pg = require('pg');
@@ -33,6 +35,8 @@ module.exports = Class.create({
 		query_timeout: 6000,
 		connectionTimeoutMillis: 30000,
 		idleTimeoutMillis: 10000,
+		passwordPlugin: "",
+		passwordPluginTimeout: 30000,
 		table: "items"
 	},
 	
@@ -63,7 +67,7 @@ module.exports = Class.create({
 		var cache_opts = pg_config.cache;
 		if (cache_opts && cache_opts.enabled) {
 			this.logDebug(3, "Setting up LRU cache", cache_opts);
-			this.cache = new Cache( Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) );
+			this.cache = new Cache({ copyBuffers: true, ...Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) });
 			this.cache.on('expire', function(item, reason) {
 				self.logDebug(9, "Expiring LRU cache object: " + item.key + " due to: " + reason, {
 					key: item.key,
@@ -75,9 +79,16 @@ module.exports = Class.create({
 		}
 		
 		// pass entire config to pg.Pool, sans our custom keys
-		var db = this.db = new pg.Pool( 
-			Tools.copyHashRemoveKeys( pg_config, { cache: 1, table: 1 }) 
-		);
+		var pool_config = Tools.copyHashRemoveKeys( pg_config, { cache: 1, table: 1, passwordPlugin: 1, passwordPluginTimeout: 1 } );
+		
+		if (pg_config.passwordPlugin) {
+			// node-postgres supports password as a sync/async callback, called for each
+			// new physical connection.  We use that hook to ask an external command for
+			// short-lived provider tokens without baking provider-specific code in here.
+			pool_config.password = this.getPasswordFromPlugin.bind(this);
+		}
+		
+		var db = this.db = new pg.Pool( pool_config );
 		
 		db.on('error', function(err) {
 			self.logError('db', "DB Error: " + err, err);
@@ -104,6 +115,89 @@ module.exports = Class.create({
 		this.queryWithClient( this.db, sql, args, callback );
 	},
 	
+	getPasswordFromPlugin: function() {
+		// execute an external command to resolve the Postgres password
+		// the plugin may also return a TTL, allowing us to cache it in memory
+		var self = this;
+		var pg_config = this.config.get();
+		var now = Date.now();
+		
+		if (this.passwordPluginCache && (now < this.passwordPluginCache.expires)) {
+			return Promise.resolve( this.passwordPluginCache.password );
+		}
+		
+		// If a plugin call is already in-flight, return the same Promise.
+		// This coalesces simultaneous pool reconnects into one plugin exec.
+		if (this.passwordPluginFetch) return this.passwordPluginFetch;
+		
+		var hook_args = {
+			type: 'postgres_password',
+			config: Tools.copyHashRemoveKeys( pg_config, { password: 1, passwordPlugin: 1, passwordPluginTimeout: 1 } )
+		};
+		
+		var child_cmd = pg_config.passwordPlugin;
+		var child_opts = {
+			cwd: os.tmpdir(),
+			timeout: pg_config.passwordPluginTimeout || 30000
+		};
+		
+		this.logDebug(5, "Calling Postgres password plugin");
+		
+		this.passwordPluginFetch = new Promise( function(resolve, reject) {
+			var child = cp.exec( child_cmd, child_opts, function(err, stdout, stderr) {
+				var json = null;
+				
+				if (!err && stdout.match(/\S/)) {
+					// parse last line only, to omit any noise from plugin
+					try { json = JSON.parse( stdout.replace(/\r\n/g, "\n").trim().split(/\n/).pop() ); }
+					catch (e) {
+						err = new Error("Postgres password plugin JSON parse error: " + (e.message || e));
+						err.code = 'json';
+					}
+				}
+				
+				if (stderr && stderr.match(/\S/)) {
+					self.logDebug(9, "Postgres password plugin emitted STDERR: " + Buffer.byteLength(stderr) + " bytes");
+				}
+				
+				if (err) return reject( err );
+				if (!json) return reject( new Error("Postgres password plugin error: No JSON found in response STDOUT") );
+				if (json.code) return reject( new Error("Postgres password plugin error: " + (json.description || json.code)) );
+				if (typeof(json.password) != 'string') return reject( new Error("Postgres password plugin error: No password string found in response JSON") );
+				
+				var ttl = parseInt( json.ttl || 0, 10 ) || 0;
+				
+				if (ttl > 0) {
+					self.passwordPluginCache = {
+						password: json.password,
+						expires: Date.now() + (ttl * 1000)
+					};
+					
+					self.logDebug(6, "Postgres password plugin returned password with TTL: " + ttl + " seconds");
+				}
+				else {
+					self.passwordPluginCache = null;
+					self.logDebug(6, "Postgres password plugin returned password without TTL");
+				}
+				
+				resolve( json.password );
+			} ); // cp.exec
+			
+			// Write hook data to child's stdin, using one compact JSON document.
+			child.stdin.on('error', noop);
+			child.stdin.write( JSON.stringify(hook_args) + "\n" );
+			child.stdin.end();
+		} ).then( function(password) {
+			self.passwordPluginFetch = null;
+			return password;
+		}, function(err) {
+			self.passwordPluginFetch = null;
+			throw err;
+		} );
+		
+		return this.passwordPluginFetch;
+	},
+	
 	queryWithClient: function(client, sql, args, callback) {
 		// run and log db query, and error if applicable
 		// allow for named variables in sql, expand to numbered $ variables for pg

package/engines/Redis.js

@@ -54,7 +54,7 @@ module.exports = Class.create({
 		var cache_opts = r_config.cache;
 		if (cache_opts && cache_opts.enabled) {
 			this.logDebug(3, "Setting up LRU cache", cache_opts);
-			this.cache = new Cache( Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) );
+			this.cache = new Cache({ copyBuffers: true, ...Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) });
 			this.cache.on('expire', function(item, reason) {
 				self.logDebug(9, "Expiring LRU cache object: " + item.key + " due to: " + reason, {
 					key: item.key,

package/engines/RedisCluster.js

@@ -56,7 +56,7 @@ module.exports = Class.create({
 		var cache_opts = r_config.cache;
 		if (cache_opts && cache_opts.enabled) {
 			this.logDebug(3, "Setting up LRU cache", cache_opts);
-			this.cache = new Cache( Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) );
+			this.cache = new Cache({ copyBuffers: true, ...Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) });
 			this.cache.on('expire', function(item, reason) {
 				self.logDebug(9, "Expiring LRU cache object: " + item.key + " due to: " + reason, {
 					key: item.key,

package/engines/S3.js

@@ -49,7 +49,7 @@ module.exports = Class.create({
 		var cache_opts = s3_config.cache;
 		if (cache_opts && cache_opts.enabled) {
 			this.logDebug(3, "Setting up LRU cache", cache_opts);
-			this.cache = new Cache( Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) );
+			this.cache = new Cache({ copyBuffers: true, ...Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) });
 			this.cache.on('expire', function(item, reason) {
 				self.logDebug(9, "Expiring LRU cache object: " + item.key + " due to: " + reason, {
 					key: item.key,

package/engines/SQLite.js

@@ -68,7 +68,7 @@ module.exports = Class.create({
 		var cache_opts = this.config.get('cache');
 		if (cache_opts && cache_opts.enabled) {
 			this.logDebug(3, "Setting up LRU cache", cache_opts);
-			this.cache = new Cache( Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) );
+			this.cache = new Cache({ copyBuffers: true, ...Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) });
 			this.cache.on('expire', function(item, reason) {
 				self.logDebug(9, "Expiring LRU cache object: " + item.key + " due to: " + reason, {
 					key: item.key,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "4.2.1",
+	"version": "4.3.0",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -30,7 +30,7 @@
 		"he": "1.1.1",
 		"moo": "0.4.3",
 		"nearley": "2.20.1",
-		"pixl-cache": "^1.0.6",
+		"pixl-cache": "^1.1.2",
 		"pixl-class": "^1.0.3",
 		"pixl-perf": "^1.0.9",
 		"pixl-server": "^1.0.39",

package/storage.js

@@ -516,6 +516,35 @@ module.exports = Class.create({
 		);
 	},
 	
+	deleteAll: function(keys, callback) {
+		// delete multiple records at once, given array of keys
+		// continue through all keys, even if some deletes fail
+		var self = this;
+		if (!this.started) return callback( new Error("Storage has not completed startup.") );
+		
+		// if engine provides its own deleteAll, call that directly
+		if (("deleteAll" in this.engine) && !this.currentTransactionPath) {
+			return this.engine.deleteAll(keys, callback);
+		}
+		
+		async.eachLimit(keys, this.concurrency, 
+			function(key, callback) {
+				// iterator for each key
+				self.delete(key, function(err) {
+					if (err) {
+						// log and swallow the error so the remaining deletes can proceed
+						self.logError('storage', "Failed to delete key in deleteAll: " + key + ": " + err);
+					}
+					callback();
+				} );
+			}, 
+			function() {
+				// all keys attempted
+				callback();
+			}
+		);
+	},
+	
 	copy: function(old_key, new_key, callback) {
 		// copy record to new key
 		var self = this;

package/test/test-main.js

@@ -656,6 +656,38 @@ module.exports = {
 			} );
 		},
 		
+		function testDeleteAll(test) {
+			// delete multiple keys using deleteAll, continuing past errors
+			test.expect(5);
+			var self = this;
+			var keys = ['deleteall1', 'deleteall_missing', 'deleteall2'];
+			
+			this.storage.putMulti( {
+				deleteall1: { fruit: 'apple' },
+				deleteall2: { fruit: 'banana' }
+			}, function(err) {
+				test.ok( !err, "No error creating deleteAll test records: " + err );
+				
+				self.storage.deleteAll( keys, function(err) {
+					test.ok( !err, "No error calling deleteAll: " + err );
+					
+					// the missing middle key should not prevent the final key from deleting
+					self.storage.get( 'deleteall1', function(err) {
+						test.ok( !!err, "Expected error fetching first deleteAll key after delete" );
+						
+						self.storage.get( 'deleteall_missing', function(err) {
+							test.ok( !!err, "Expected error fetching missing deleteAll key after delete" );
+							
+							self.storage.get( 'deleteall2', function(err) {
+								test.ok( !!err, "Expected error fetching final deleteAll key after delete" );
+								test.done();
+							} );
+						} );
+					} );
+				} );
+			} );
+		},
+		
 		function testOptimize(test) {
 			// run engine-specific optimization, or a safe no-op if unsupported
 			test.expect(8);

package/test/test-postgres-password-plugin.js

@@ -0,0 +1,243 @@
+// Unit tests for Postgres password plugin support
+// Copyright (c) 2026 Joseph Huckaby
+// Released under the MIT License
+
+var cp = require('child_process');
+var EventEmitter = require('events').EventEmitter;
+var Module = require('module');
+
+var origLoad = Module._load;
+var origExec = cp.exec;
+
+// The Postgres engine requires 'pg' at module load time.  Stub it here so this
+// focused test can run without needing a live Postgres module or database.
+var fakePG = {
+	lastConfig: null,
+	Pool: function(config) {
+		fakePG.lastConfig = config;
+		this.on = function() {};
+		this.query = function(sql, args, callback) {
+			callback( null, { command: 'CREATE', rowCount: 0, rows: [] } );
+		};
+	}
+};
+
+Module._load = function(request, parent, isMain) {
+	if (request == 'pg') return fakePG;
+	return origLoad.apply( this, arguments );
+};
+
+var PostgresEngine = require('../engines/Postgres.js');
+Module._load = origLoad;
+
+function makeEngine(config) {
+	// Keep this intentionally tiny.  The password plugin helper only needs the
+	// component config and log methods, and setup() only needs query logging.
+	var engine = Object.create( PostgresEngine.prototype );
+	engine.config = {
+		get: function() { return config; }
+	};
+	engine.logDebug = function() {};
+	engine.logError = function() {};
+	return engine;
+}
+
+function mockExec(responder) {
+	var calls = [];
+	
+	cp.exec = function(cmd, opts, callback) {
+		var call = { cmd: cmd, opts: opts, stdin: '' };
+		var child = { stdin: new EventEmitter() };
+		calls.push( call );
+		
+		child.stdin.write = function(data) {
+			call.stdin += data;
+		};
+		child.stdin.end = function() {
+			responder( call, callback );
+		};
+		
+		return child;
+	};
+	
+	return calls;
+}
+
+function restoreExec() {
+	cp.exec = origExec;
+}
+
+module.exports = {
+	tests: [
+		
+		function setup_installsPasswordFunctionAndStripsPluginKeys(test) {
+			test.expect(4);
+			
+			var engine = makeEngine({
+				host: 'localhost',
+				database: 'test',
+				user: 'test',
+				password: 'static',
+				passwordPlugin: 'node ./plugin.js',
+				passwordPluginTimeout: 1234,
+				table: 'items'
+			});
+			
+			fakePG.lastConfig = null;
+			engine.setup( function(err) {
+				test.ok( !err, "Setup completed without error" );
+				test.ok( typeof(fakePG.lastConfig.password) == 'function', "Password was converted to a callback function" );
+				test.ok( !('passwordPlugin' in fakePG.lastConfig), "passwordPlugin was stripped from pool config" );
+				test.ok( !('passwordPluginTimeout' in fakePG.lastConfig), "passwordPluginTimeout was stripped from pool config" );
+				test.done();
+			} );
+		},
+		
+		function passwordPlugin_returnsPasswordAndSendsConfig(test) {
+			test.expect(5);
+			
+			var engine = makeEngine({
+				host: 'localhost',
+				database: 'test',
+				user: 'tester',
+				password: 'static',
+				passwordPlugin: 'node ./plugin.js',
+				passwordPluginTimeout: 1234,
+				table: 'items'
+			});
+			
+			var calls = mockExec( function(call, callback) {
+				callback( null, JSON.stringify({ xy: 1, password: 'dynamic-secret' }) + "\n", "" );
+			} );
+			
+			engine.getPasswordFromPlugin().then( function(password) {
+				var input = JSON.parse( calls[0].stdin );
+				
+				test.ok( password == 'dynamic-secret', "Returned dynamic password" );
+				test.ok( calls[0].cmd == 'node ./plugin.js', "Used configured plugin command" );
+				test.ok( calls[0].opts.timeout == 1234, "Used configured plugin timeout" );
+				test.ok( input.type == 'postgres_password', "Sent expected hook type" );
+				test.ok( !('password' in input.config), "Static password was omitted from plugin input" );
+				
+				restoreExec();
+				test.done();
+			} ).catch( function(err) {
+				restoreExec();
+				test.ok( false, "Unexpected error: " + err );
+				test.done();
+			} );
+		},
+		
+		function passwordPlugin_cachesPasswordForTTL(test) {
+			test.expect(3);
+			
+			var engine = makeEngine({
+				passwordPlugin: 'node ./plugin.js',
+				table: 'items'
+			});
+			
+			var calls = mockExec( function(call, callback) {
+				callback( null, JSON.stringify({ xy: 1, password: 'cached-secret', ttl: 60 }) + "\n", "" );
+			} );
+			
+			engine.getPasswordFromPlugin().then( function(first) {
+				return engine.getPasswordFromPlugin().then( function(second) {
+					test.ok( first == 'cached-secret', "First password was returned" );
+					test.ok( second == 'cached-secret', "Second password was returned from cache" );
+					test.ok( calls.length == 1, "Plugin was only invoked once" );
+					
+					restoreExec();
+					test.done();
+				} );
+			} ).catch( function(err) {
+				restoreExec();
+				test.ok( false, "Unexpected error: " + err );
+				test.done();
+			} );
+		},
+		
+		function passwordPlugin_doesNotCacheWithoutTTL(test) {
+			test.expect(2);
+			
+			var counter = 0;
+			var engine = makeEngine({
+				passwordPlugin: 'node ./plugin.js',
+				table: 'items'
+			});
+			
+			var calls = mockExec( function(call, callback) {
+				counter++;
+				callback( null, JSON.stringify({ xy: 1, password: 'secret-' + counter }) + "\n", "" );
+			} );
+			
+			engine.getPasswordFromPlugin().then( function(first) {
+				return engine.getPasswordFromPlugin().then( function(second) {
+					test.ok( first != second, "Passwords were fetched separately" );
+					test.ok( calls.length == 2, "Plugin was invoked twice" );
+					
+					restoreExec();
+					test.done();
+				} );
+			} ).catch( function(err) {
+				restoreExec();
+				test.ok( false, "Unexpected error: " + err );
+				test.done();
+			} );
+		},
+		
+		function passwordPlugin_coalescesConcurrentFetches(test) {
+			test.expect(2);
+			
+			var engine = makeEngine({
+				passwordPlugin: 'node ./plugin.js',
+				table: 'items'
+			});
+			
+			var calls = mockExec( function(call, callback) {
+				setTimeout( function() {
+					callback( null, JSON.stringify({ xy: 1, password: 'shared-secret', ttl: 60 }) + "\n", "" );
+				}, 10 );
+			} );
+			
+			Promise.all([
+				engine.getPasswordFromPlugin(),
+				engine.getPasswordFromPlugin(),
+				engine.getPasswordFromPlugin()
+			]).then( function(passwords) {
+				test.ok( passwords.join(',') == 'shared-secret,shared-secret,shared-secret', "All callers received the same password" );
+				test.ok( calls.length == 1, "Only one plugin process was spawned" );
+				
+				restoreExec();
+				test.done();
+			} ).catch( function(err) {
+				restoreExec();
+				test.ok( false, "Unexpected error: " + err );
+				test.done();
+			} );
+		},
+		
+		function passwordPlugin_reportsPluginError(test) {
+			test.expect(1);
+			
+			var engine = makeEngine({
+				passwordPlugin: 'node ./plugin.js',
+				table: 'items'
+			});
+			
+			mockExec( function(call, callback) {
+				callback( null, JSON.stringify({ xy: 1, code: 'auth', description: 'Token rejected' }) + "\n", "" );
+			} );
+			
+			engine.getPasswordFromPlugin().then( function() {
+				restoreExec();
+				test.ok( false, "Expected plugin error" );
+				test.done();
+			} ).catch( function(err) {
+				restoreExec();
+				test.ok( /Token rejected/.test(err.message), "Reported plugin error: " + err.message );
+				test.done();
+			} );
+		}
+		
+	]
+};

package/transaction.js

@@ -575,7 +575,8 @@ module.exports = Class.create({
 		trans.aborting = true;
 		
 		var num_actions = Tools.numKeys(trans.keys || {});
-		this.logError('rollback', "Aborting transaction: " + trans.id, { path: path, actions: num_actions });
+		if (num_actions) this.logError('rollback', "Aborting transaction: " + trans.id, { path: path, actions: num_actions });
+		else this.logDebug(6, "Aborting empty transaction (no-op): " + trans.id, { path: path, actions: num_actions });
 		
 		// read in file line by line
 		// (file may not exist, which is fine, hence 'ignore_not_found')