pixl-server-storage 3.0.23 → 3.1.0

package/README.md

@@ -252,6 +252,12 @@ The `expiration_updates` property activates additional features in the [expirati
 
 Note that this feature incurs additional overhead, because the expiration date of every record needs to be stored in a global [Hash](docs/Hashes.md).  This slows down both the expiration set operation, and the nightly maintenance sweep to delete expired records.  For this reason, the `expiration_dates` property defaults to `false` (disabled).
 
+## lower_case_keys
+
+The `lower_case_keys` property causes all storage keys to be internally lower-cased, effectively making all storage paths case-insensitive.  This is the default (`true`).  If you set this property to `false`, then all storage keys retain their natural casing, effectively making them case-sensitive.
+
+Please note that if you use the [Local Filesystem](#local-filesystem) engine, then the filesystem itself may be case-insensitive (e.g. legacy macOS HFS).
+
 ## debug (standalone)
 
 The `debug` property is only used when using [Standalone Mode](#standalone-mode).  Setting this to `true` will cause the engine to emit debugging messages to the console.
@@ -409,27 +415,18 @@ Note that binary records are **not** cached.  This system is for JSON records on
 
 ## Amazon S3
 
-If you want to use [Amazon S3](http://aws.amazon.com/s3/) as a backing store, here is how to do so.  First, you need to manually install the [aws-sdk](https://www.npmjs.com/package/aws-sdk) module into your app:
-
-```
-npm install --save aws-sdk
-```
-
-Then configure your storage thusly:
+If you want to use [Amazon S3](http://aws.amazon.com/s3/) as a backing store, configure your storage thusly:
 
 ```javascript
 {
 	"engine": "S3",
 	"AWS": {
-		"accessKeyId": "YOUR_AMAZON_ACCESS_KEY", 
-		"secretAccessKey": "YOUR_AMAZON_SECRET_KEY", 
 		"region": "us-west-1",
-		"correctClockSkew": true,
-		"maxRetries": 5,
-		"httpOptions": {
-			"connectTimeout": 5000,
-			"timeout": 5000
-		}
+		"credentials": {
+			"accessKeyId": "YOUR_AMAZON_ACCESS_KEY", 
+			"secretAccessKey": "YOUR_AMAZON_SECRET_KEY", 
+		},
+		"maxAttempts": 5
 	},
 	"S3": {
 		"keyPrefix": "",
@@ -448,10 +445,6 @@ Then configure your storage thusly:
 
 Replace `YOUR_AMAZON_ACCESS_KEY` and `YOUR_AMAZON_SECRET_KEY` with your Amazon Access Key and Secret Key, respectively.  These can be generated on the Security Credentials page.  Replace `MY_S3_BUCKET_ID` with the ID if your own S3 bucket.  Make sure you match up the region too.
 
-The `AWS` object is passed directly to the `config.update()` function from the [aws-sdk](https://www.npmjs.com/package/aws-sdk) module, so you can also include any properties supported there.  See the [docs](http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/node-configuring.html) for more details.
-
-The `S3` object is passed directly to the S3 class constructor, so check the [docs](http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#constructor-property) to see what other properties are supported.   There are a few special `S3` properties used by pixl-server-storage, which are described below.
-
 If you plan on using Amazon AWS in other parts of your application, you can actually move the `AWS` config object into your outer server configuration.  The storage module will look for it there.
 
 ### S3 File Extensions
@@ -495,8 +488,6 @@ Besides hash marks, the special macro `[key]` will be substituted with the full
 
 It is *highly* recommended that you enable caching for S3, which keeps a copy of the most recently used JSON records in RAM.  Not only will this increase overall performance, but it is especially important if you use any of the advanced storage features like [Lists](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Lists.md), [Hashes](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Hashes.md), [Transactions](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Transactions.md) or the [Indexer](https://github.com/jhuckaby/pixl-server-storage/blob/master/docs/Indexer.md).
 
-The reason why caching is so important is because Amazon S3 is only [eventually consistent](https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel) when replacing existing records, so corruption can occur when writing compound objects (i.e. a list or a hash) and then quickly reading them back in.  To protect against this, all writes can be cached in RAM, so subsequent reads can often bypass S3 and just read from the RAM cache instead, ensuring that the correct (latest) data is read.
-
 To enable the S3 cache, include a `cache` object in your `S3` configuration with the following properties:
 
 ```js

package/engines/S3.js

@@ -1,15 +1,13 @@
 // Amazon AWS S3 Storage Plugin
-// Copyright (c) 2015 - 2019 Joseph Huckaby
+// Copyright (c) 2015 - 2022 Joseph Huckaby
 // Released under the MIT License
 
-// Requires the 'aws-sdk' module from npm
-// npm install aws-sdk
-
 var Class = require("pixl-class");
 var Component = require("pixl-server/component");
 var Tools = require("pixl-tools");
-var AWS = require('aws-sdk');
 var Cache = require("pixl-cache");
+var S3 = require("@aws-sdk/client-s3");
+var streamToBuffer = require("fast-stream-to-buffer");
 
 module.exports = Class.create({
 	
@@ -41,13 +39,6 @@ module.exports = Class.create({
 		this.keyTemplate = (s3_config.keyTemplate || '').replace(/^\//, '').replace(/\/$/, '');
 		this.fileExtensions = !!s3_config.fileExtensions;
 		
-		if (this.debugLevel(10)) {
-			// S3 has a logger API but it's extremely verbose -- restrict to level 10 only
-			s3_config.logger = {
-				log: function(msg) { self.logDebug(10, "S3 Debug: " + msg); }
-			};
-		}
-		
 		// optional LRU cache
 		this.cache = null;
 		var cache_opts = s3_config.cache;
@@ -63,10 +54,36 @@ module.exports = Class.create({
 				});
 			});
 		}
-		delete s3_config.cache;
 		
-		AWS.config.update( aws_config );
-		this.s3 = new AWS.S3( Tools.copyHashRemoveKeys(s3_config, { keyPrefix:1, keyTemplate:1, fileExtensions:1, cache:1 }) );
+		// merge AWS and S3 configs
+		var combo_config = Tools.mergeHashes( aws_config, s3_config );
+		
+		// convert v2 config to v3
+		if (!combo_config.maxAttempts && combo_config.maxRetries) {
+			combo_config.maxAttempts = combo_config.maxRetries;
+			delete combo_config.maxRetries;
+		}
+		if (combo_config.accessKeyId) {
+			if (!combo_config.credentials) combo_config.credentials = {};
+			combo_config.credentials.accessKeyId = combo_config.accessKeyId;
+			delete combo_config.accessKeyId;
+		}
+		if (combo_config.secretAccessKey) {
+			if (!combo_config.credentials) combo_config.credentials = {};
+			combo_config.credentials.secretAccessKey = combo_config.secretAccessKey;
+			delete combo_config.secretAccessKey;
+		}
+		delete combo_config.correctClockSkew;
+		delete combo_config.httpOptions;
+		delete combo_config.keyPrefix;
+		delete combo_config.keyTemplate;
+		delete combo_config.fileExtensions;
+		delete combo_config.cache;
+		
+		this.s3Params = combo_config.params || {};
+		delete combo_config.params;
+		
+		this.s3 = new S3.S3Client(combo_config);
 	},
 	
 	prepKey: function(key) {
@@ -107,7 +124,7 @@ module.exports = Class.create({
 		var is_binary = this.storage.isBinaryKey(key);
 		key = this.prepKey(key);
 		
-		var params = {};
+		var params = Tools.copyHash( this.s3Params );
 		params.Key = this.extKey(key, orig_key);
 		params.Body = value;
 		
@@ -121,23 +138,21 @@ module.exports = Class.create({
 			params.ContentType = 'application/json';
 		}
 		
-		// double-callback protection (bug in aws-sdk)
-		var done = false;
-		this.s3.putObject( params, function(err, data) {
-			if (done) return; else done = true;
-			
-			if (err) {
+		this.s3.send( new S3.PutObjectCommand(params) )
+			.then( function(data) {
+				self.logDebug(9, "Store complete: " + key);
+				
+				// possibly cache in LRU
+				if (self.cache && !is_binary) {
+					self.cache.set( orig_key, params.Body, { date: Tools.timeNow(true) } );
+				}
+				
+				if (callback) callback(null, data);
+			} )
+			.catch( function(err) {
 				self.logError('s3', "Failed to store object: " + key + ": " + (err.message || err), err);
-			}
-			else self.logDebug(9, "Store complete: " + key);
-			
-			// possibly cache in LRU
-			if (self.cache && !is_binary) {
-				self.cache.set( orig_key, params.Body, { date: Tools.timeNow(true) } );
-			}
-			
-			if (callback) callback(err, data);
-		} );
+				if (callback) callback(err);
+			} );
 	},
 	
 	putStream: function(key, inp, callback) {
@@ -146,24 +161,21 @@ module.exports = Class.create({
 		var orig_key = key;
 		key = this.prepKey(key);
 		
-		var params = {};
+		var params = Tools.copyHash( this.s3Params );
 		params.Key = this.extKey(key, orig_key);
 		params.Body = inp;
 		
 		this.logDebug(9, "Storing S3 Binary Stream: " + key);
 		
-		// double-callback protection (bug in aws-sdk)
-		var done = false;
-		this.s3.upload(params, function(err, data) {
-			if (done) return; else done = true;
-			
-			if (err) {
+		this.s3.send( new S3.PutObjectCommand(params) )
+			.then( function(data) {
+				self.logDebug(9, "Stream store complete: " + key);
+				if (callback) callback(null, data);
+			} )
+			.catch( function(err) {
 				self.logError('s3', "Failed to store stream: " + key + ": " + (err.message || err), err);
-			}
-			else self.logDebug(9, "Stream store complete: " + key);
-			
-			if (callback) callback(err, data);
-		} );
+				if (callback) callback(err);
+			} );
 	},
 	
 	putStreamCustom: function(key, inp, opts, callback) {
@@ -172,25 +184,22 @@ module.exports = Class.create({
 		var orig_key = key;
 		key = this.prepKey(key);
 		
-		var params = {};
+		var params = Tools.copyHash( this.s3Params );
 		params.Key = this.extKey(key, orig_key);
 		params.Body = inp;
 		if (opts) Tools.mergeHashInto(params, opts);
 		
 		this.logDebug(9, "Storing S3 Binary Stream: " + key);
 		
-		// double-callback protection (bug in aws-sdk)
-		var done = false;
-		this.s3.upload(params, function(err, data) {
-			if (done) return; else done = true;
-			
-			if (err) {
+		this.s3.send( new S3.PutObjectCommand(params) )
+			.then( function(data) {
+				self.logDebug(9, "Stream store complete: " + key);
+				if (callback) callback(null, data);
+			} )
+			.catch( function(err) {
 				self.logError('s3', "Failed to store stream: " + key + ": " + (err.message || err), err);
-			}
-			else self.logDebug(9, "Stream store complete: " + key);
-			
-			if (callback) callback(err, data);
-		} );
+				if (callback) callback(err);
+			} );
 	},
 	
 	head: function(key, callback) {
@@ -214,13 +223,20 @@ module.exports = Class.create({
 			return;
 		} // cache
 		
-		// double-callback protection (bug in aws-sdk)
-		var done = false;
-		this.s3.headObject( { Key: this.extKey(key, orig_key) }, function(err, data) {
-			if (done) return; else done = true;
-			
-			if (err) {
-				if ((err.code == 'NoSuchKey') || (err.code == 'NotFound')) {
+		var params = Tools.copyHash( this.s3Params );
+		params.Key = this.extKey(key, orig_key);
+		
+		this.s3.send( new S3.HeadObjectCommand(params) )
+			.then( function(data) {
+				self.logDebug(9, "Head complete: " + key);
+				
+				callback( null, {
+					mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
+					len: data.ContentLength
+				} );
+			} )
+			.catch( function(err) {
+				if ((err.name == 'NoSuchKey') || (err.name == 'NotFound') || (err.code == 'NoSuchKey') || (err.code == 'NotFound')) {
 					// key not found, special case, don't log an error
 					// always include "Not found" in error message
 					err = new Error("Failed to head key: " + key + ": Not found");
@@ -230,16 +246,8 @@ module.exports = Class.create({
 					// some other error
 					self.logError('s3', "Failed to head key: " + key + ": " + (err.message || err), err);
 				}
-				callback( err, null );
-				return;
-			}
-			
-			self.logDebug(9, "Head complete: " + key);
-			callback( null, {
-				mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
-				len: data.ContentLength
+				callback( err );
 			} );
-		} );
 	},
 	
 	get: function(key, callback) {
@@ -269,13 +277,46 @@ module.exports = Class.create({
 			return;
 		} // cache
 		
-		// double-callback protection (bug in aws-sdk)
-		var done = false;
-		this.s3.getObject( { Key: this.extKey(key, orig_key) }, function(err, data) {
-			if (done) return; else done = true;
-			
-			if (err) {
-				if ((err.code == 'NoSuchKey') || (err.code == 'NotFound')) {
+		var params = Tools.copyHash( this.s3Params );
+		params.Key = this.extKey(key, orig_key);
+		
+		this.s3.send( new S3.GetObjectCommand(params) )
+			.then( function(data) {
+				// stream to buffer
+				streamToBuffer( data.Body, function (err, body) {
+					if (err) {
+						self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
+						return callback(err);
+					}
+					
+					if (is_binary) {
+						self.logDebug(9, "Binary fetch complete: " + key, '' + body.length + ' bytes');
+					}
+					else {
+						body = body.toString();
+						
+						// possibly cache in LRU
+						if (self.cache) {
+							self.cache.set( orig_key, body, { date: Tools.timeNow(true) } );
+						}
+						
+						try { body = JSON.parse( body ); }
+						catch (e) {
+							self.logError('s3', "Failed to parse JSON record: " + key + ": " + e);
+							callback( e, null );
+							return;
+						}
+						self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? body : null);
+					}
+					
+					callback( null, body, {
+						mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
+						len: data.ContentLength
+					} );
+				} ); // streamToBuffer
+			} )
+			.catch( function(err) {
+				if ((err.name == 'NoSuchKey') || (err.name == 'NotFound') || (err.code == 'NoSuchKey') || (err.code == 'NotFound')) {
 					// key not found, special case, don't log an error
 					// always include "Not found" in error message
 					err = new Error("Failed to fetch key: " + key + ": Not found");
@@ -285,37 +326,8 @@ module.exports = Class.create({
 					// some other error
 					self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
 				}
-				callback( err, null );
-				return;
-			}
-			
-			var body = null;
-			if (is_binary) {
-				body = data.Body;
-				self.logDebug(9, "Binary fetch complete: " + key, '' + body.length + ' bytes');
-			}
-			else {
-				body = data.Body.toString();
-				
-				// possibly cache in LRU
-				if (self.cache) {
-					self.cache.set( orig_key, body, { date: Tools.timeNow(true) } );
-				}
-				
-				try { body = JSON.parse( body ); }
-				catch (e) {
-					self.logError('s3', "Failed to parse JSON record: " + key + ": " + e);
-					callback( e, null );
-					return;
-				}
-				self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? body : null);
-			}
-			
-			callback( null, body, {
-				mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
-				len: data.ContentLength
+				callback( err );
 			} );
-		} );
 	},
 	
 	getStream: function(key, callback) {
@@ -326,27 +338,30 @@ module.exports = Class.create({
 		
 		this.logDebug(9, "Fetching S3 Stream: " + key);
 		
-		var params = { Key: this.extKey(key, orig_key) };
-		var download = self.s3.getObject(params).createReadStream();
-		var proceed = false;
-		
-		download.on('error', function(err) {
-			if (proceed) self.logError('s3', "Failed to download key: " + key + ": " + (err.message || err), err);
-		});
-		download.once('end', function() {
-			self.logDebug(9, "S3 stream download complete: " + key);
-		} );
-		download.once('close', function() {
-			self.logDebug(9, "S3 stream download closed: " + key);
-		} );
-		
-		// double-callback protection (bug in aws-sdk)
-		var done = false;
-		this.s3.headObject( params, function(err, data) {
-			if (done) return; else done = true;
-			
-			if (err) {
-				if ((err.code == 'NoSuchKey') || (err.code == 'NotFound')) {
+		var params = Tools.copyHash( this.s3Params );
+		params.Key = this.extKey(key, orig_key);
+		
+		this.s3.send( new S3.GetObjectCommand(params) )
+			.then( function(data) {
+				var download = data.Body;
+				
+				download.on('error', function(err) {
+					self.logError('s3', "Failed to download key: " + key + ": " + (err.message || err), err);
+				});
+				download.once('end', function() {
+					self.logDebug(9, "S3 stream download complete: " + key);
+				} );
+				download.once('close', function() {
+					self.logDebug(9, "S3 stream download closed: " + key);
+				} );
+				
+				callback( null, download, {
+					mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
+					len: data.ContentLength
+				} );
+			})
+			.catch( function(err) {
+				if ((err.name == 'NoSuchKey') || (err.name == 'NotFound') || (err.code == 'NoSuchKey') || (err.code == 'NotFound')) {
 					// key not found, special case, don't log an error
 					// always include "Not found" in error message
 					err = new Error("Failed to fetch key: " + key + ": Not found");
@@ -356,18 +371,8 @@ module.exports = Class.create({
 					// some other error
 					self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
 				}
-				
-				download.destroy();
-				callback( err, null );
-				return;
-			}
-			
-			proceed = true;
-			callback( null, download, {
-				mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
-				len: data.ContentLength
-			} );
-		}); // headObject
+				callback( err );
+			});
 	},
 	
 	getStreamRange: function(key, start, end, callback) {
@@ -378,7 +383,8 @@ module.exports = Class.create({
 		
 		this.logDebug(9, "Fetching ranged S3 stream: " + key, { start, end });
 		
-		var params = { Key: this.extKey(key, orig_key) };
+		var params = Tools.copyHash( this.s3Params );
+		params.Key = this.extKey(key, orig_key);
 		
 		// convert start/end to HTTP range header string
 		var range = "bytes=";
@@ -386,26 +392,29 @@ module.exports = Class.create({
 		range += '-';
 		if (!isNaN(end)) range += end;
 		
-		var download = self.s3.getObject( Tools.mergeHashes(params, { Range: range }) ).createReadStream();
-		var proceed = false;
-		
-		download.on('error', function(err) {
-			if (proceed) self.logError('s3', "Failed to download key: " + key + ": " + (err.message || err), err);
-		});
-		download.once('end', function() {
-			self.logDebug(9, "S3 stream download complete: " + key);
-		} );
-		download.once('close', function() {
-			self.logDebug(9, "S3 stream download closed: " + key);
-		} );
-		
-		// double-callback protection (bug in aws-sdk)
-		var done = false;
-		this.s3.headObject( params, function(err, data) {
-			if (done) return; else done = true;
-			
-			if (err) {
-				if ((err.code == 'NoSuchKey') || (err.code == 'NotFound')) {
+		params.Range = range;
+		
+		this.s3.send( new S3.GetObjectCommand(params) )
+			.then( function(data) {
+				var download = data.Body;
+				
+				download.on('error', function(err) {
+					self.logError('s3', "Failed to download key: " + key + ": " + (err.message || err), err);
+				});
+				download.once('end', function() {
+					self.logDebug(9, "S3 stream download complete: " + key);
+				} );
+				download.once('close', function() {
+					self.logDebug(9, "S3 stream download closed: " + key);
+				} );
+				
+				callback( null, download, {
+					mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
+					len: data.ContentLength
+				} );
+			})
+			.catch( function(err) {
+				if ((err.name == 'NoSuchKey') || (err.name == 'NotFound') || (err.code == 'NoSuchKey') || (err.code == 'NotFound')) {
 					// key not found, special case, don't log an error
 					// always include "Not found" in error message
 					err = new Error("Failed to fetch key: " + key + ": Not found");
@@ -415,32 +424,8 @@ module.exports = Class.create({
 					// some other error
 					self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
 				}
-				
-				download.destroy();
-				callback( err, null );
-				return;
-			}
-			
-			// validate byte range, now that we have the head info
-			if (isNaN(start) && !isNaN(end)) {
-				start = data.ContentLength - end;
-				end = data.ContentLength ? data.ContentLength - 1 : 0;
-			} 
-			else if (!isNaN(start) && isNaN(end)) {
-				end = data.ContentLength ? data.ContentLength - 1 : 0;
-			}
-			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= data.ContentLength) || (end < start) || (end >= data.ContentLength)) {
-				download.destroy();
-				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + data.ContentLength + ")"), null );
-				return;
-			}
-			
-			proceed = true;
-			callback( null, download, {
-				mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
-				len: data.ContentLength
-			} );
-		}); // headObject
+				callback( err );
+			});
 	},
 	
 	delete: function(key, callback) {
@@ -451,23 +436,24 @@ module.exports = Class.create({
 		
 		this.logDebug(9, "Deleting S3 Object: " + key);
 		
-		// double-callback protection (bug in aws-sdk)
-		var done = false;
-		this.s3.deleteObject( { Key: this.extKey(key, orig_key) }, function(err, data) {
-			if (done) return; else done = true;
-			
-			if (err) {
+		var params = Tools.copyHash( this.s3Params );
+		params.Key = this.extKey(key, orig_key);
+		
+		this.s3.send( new S3.DeleteObjectCommand(params) )
+			.then( function(data) {
+				self.logDebug(9, "Delete complete: " + key);
+				
+				// possibly delete from LRU cache as well
+				if (self.cache && self.cache.has(orig_key)) {
+					self.cache.delete(orig_key);
+				}
+				
+				if (callback) callback(null, data);
+			} )
+			.catch( function(err) {
 				self.logError('s3', "Failed to delete object: " + key + ": " + (err.message || err), err);
-			}
-			else self.logDebug(9, "Delete complete: " + key);
-			
-			// possibly delete from LRU cache as well
-			if (self.cache && self.cache.has(orig_key)) {
-				self.cache.delete(orig_key);
-			}
-			
-			if (callback) callback(err, data);
-		} );
+				if (callback) callback(err);
+			} );
 	},
 	
 	runMaintenance: function(callback) {

package/index_types/Date.js

@@ -159,7 +159,9 @@ module.exports = Class.create({
 		}
 		
 		// load index summary for list of all populated dates
+		var dspf = state.perf.begin('date_summary');
 		this.get( sum_path, function(err, summary) {
+			dspf.end();
 			if (err || !summary) {
 				summary = { id: query.def.id, values: {} };
 			}
@@ -190,6 +192,7 @@ module.exports = Class.create({
 			}
 			
 			// now perform OR search for all applicable words
+			var drpf = state.perf.begin('date_range');
 			async.eachLimit( words, self.concurrency,
 				function(word, callback) {
 					// for each word, iterate over record ids
@@ -203,6 +206,8 @@ module.exports = Class.create({
 				},
 				function(err) {
 					// all done, perform final merge
+					drpf.end();
+					state.perf.count('date_buckets', words.length);
 					if (err) return callback(err);
 					self.mergeIndex( record_ids, temp_results, state.first ? 'or' : state.mode );
 					state.first = false;

package/index_types/Number.js

@@ -59,7 +59,7 @@ module.exports = Class.create({
 		
 		if (value.match(/^(N?)(\d+)$/i)) {
 			var neg = RegExp.$1.toUpperCase();
-			var value = parseInt( RegExp.$2 );
+			var value = Math.floor( parseInt( RegExp.$2 ) * (def.multiply || 1) / (def.divide || 1) );
 			value = Math.min( NUMBER_INDEX_MAX, value );
 			
 			var tkey = 'T' + neg + Math.floor( Math.floor(value / 1000) * 1000 );
@@ -103,7 +103,7 @@ module.exports = Class.create({
 		
 		// clean number up
 		word = word.replace(/^N/i, '-').replace(/[^\d\-]+/g, '');
-		word = '' + Math.min( NUMBER_INDEX_MAX, Math.max( NUMBER_INDEX_MIN, parseInt(word) ) );
+		word = '' + Math.min( NUMBER_INDEX_MAX, Math.max( NUMBER_INDEX_MIN, Math.floor( parseInt(word) * (query.def.multiply || 1) / (query.def.divide || 1) ) ) );
 		word = word.replace(/\-/, 'N');
 		query.word = word;
 		
@@ -119,7 +119,9 @@ module.exports = Class.create({
 		}
 		
 		// load index summary for list of all populated numbers
+		var nspf = state.perf.begin('number_summary');
 		this.get( sum_path, function(err, summary) {
+			nspf.end();
 			if (err || !summary) {
 				summary = { id: query.def.id, values: {} };
 			}
@@ -150,6 +152,7 @@ module.exports = Class.create({
 			}
 			
 			// now perform OR search for all applicable words
+			var nrpf = state.perf.begin('number_range');
 			async.eachLimit( words, self.concurrency,
 				function(word, callback) {
 					// for each word, iterate over record ids
@@ -163,6 +166,8 @@ module.exports = Class.create({
 				},
 				function(err) {
 					// all done, perform final merge
+					nrpf.end();
+					state.perf.count('number_buckets', words.length);
 					if (err) return callback(err);
 					self.mergeIndex( record_ids, temp_results, state.first ? 'or' : state.mode );
 					state.first = false;
@@ -185,7 +190,7 @@ module.exports = Class.create({
 		
 		// clean number up
 		word = word.replace(/^N/i, '-').replace(/[^\d\-]+/g, '');
-		word = '' + Math.min( NUMBER_INDEX_MAX, Math.max( NUMBER_INDEX_MIN, parseInt(word) ) );
+		word = '' + Math.min( NUMBER_INDEX_MAX, Math.max( NUMBER_INDEX_MIN, Math.floor( parseInt(word) * (query.def.multiply || 1) / (query.def.divide || 1) ) ) );
 		word = word.replace(/\-/, 'N');
 		query.word = word;
 		

package/indexer.js

@@ -5,6 +5,7 @@
 var async = require('async');
 var Class = require("pixl-class");
 var Tools = require("pixl-tools");
+var Perf = require("pixl-perf");
 var nearley = require("nearley");
 var pxql_grammar = require("./pxql.js");
 var stemmer = require('porter-stemmer').stemmer;
@@ -1220,15 +1221,16 @@ module.exports = Class.create({
 		var path = config.base_path;
 		var pf = this.perf.begin('search');
 		
+		var orig_query = query;
+		if (typeof(query) == 'object') query = Tools.copyHash(query, true);
+		
 		this.shareLock( 'C|'+path, true, function(err, lock) {
 			// got shared lock
 			self._searchRecords( query, config, function(err, results, state) {
 				// search complete
-				var elapsed = pf.end();
-				
 				if (!err) self.logTransaction('search', path, {
-					query: query,
-					elapsed_ms: elapsed,
+					query: orig_query,
+					perf: state.perf ? state.perf.metrics() : {},
 					results: (self.logEventTypes.search || self.logEventTypes.all) ? Tools.numKeys(results) : 0
 				});
 				
@@ -1251,9 +1253,15 @@ module.exports = Class.create({
 			if (query == '*') {
 				// fetch all records
 				this.logDebug(8, "Fetching all records: " + config.base_path);
+				var apf = new Perf();
+				apf.begin();
+				apf.begin('all');
+				
 				return this.hashGetAll( config.base_path + '/_id', function(err, results) {
 					// ignore error, just return empty hash
-					callback( null, results || {}, {} );
+					apf.end('all');
+					apf.end();
+					callback( null, results || {}, { perf: apf } );
 				} );
 			}
 			else if (query.match(/^\([\s\S]+\)$/)) {
@@ -1282,6 +1290,12 @@ module.exports = Class.create({
 		state.record_ids = Object.create(null);
 		state.first = true;
 		
+		// track detailed perf of search operations
+		if (!state.perf) {
+			state.perf = new Perf();
+			state.perf.begin();
+		}
+		
 		// first, split criteria into subs (sub-queries), 
 		// stds (standard queries) and negs (negative queries)
 		var subs = [], stds = [], negs = [];
@@ -1302,11 +1316,14 @@ module.exports = Class.create({
 		}
 		
 		// stds need to be weighed and sorted by weight ascending
+		var wpf = state.perf.begin('weigh');
 		async.eachLimit( (query.mode == 'and') ? stds : [], this.concurrency,
 			function(crit, callback) {
 				self.weighCriterion(crit, config, callback);
 			},
 			function(err) {
+				wpf.end();
+				
 				// sort stds by weight ascending (only needed in AND mode)
 				if (query.mode == 'and') {
 					stds = stds.sort( function(a, b) { return a.weight - b.weight; } );
@@ -1317,9 +1334,12 @@ module.exports = Class.create({
 				var tasks = [].concat( subs, stds, negs );
 				async.eachSeries( tasks,
 					function(task, callback) {
+						task.perf = state.perf;
+						
 						if (task.criteria) {
-							// sub-query
+							// sub-query	
 							self._searchRecords( task, config, function(err, records) {
+								state.perf.count('subs', 1);
 								self.mergeIndex( state.record_ids, records, state.first ? 'or' : state.mode );
 								state.first = false;
 								callback();
@@ -1333,13 +1353,31 @@ module.exports = Class.create({
 							// custom index type, e.g. date, time, number
 							var func = 'searchIndex_' + task.def.type;
 							if (!self[func]) return callback( new Error("Unknown index type: " + task.def.type) );
-							self[func]( task, state, callback );
+							
+							var cpf = state.perf.begin('search_' + task.def.id + '_' + task.def.type);
+							self[func]( task, state, function(err) {
+								cpf.end();
+								state.perf.count(task.def.type + 's', 1);
+								callback(err);
+							} );
 						}
 						else if (task.literal) {
-							self.searchWordIndexLiteral(task, state, callback);
+							// literal multi-word phrase
+							var spf = state.perf.begin('search_' + task.def.id + '_literal');
+							self.searchWordIndexLiteral(task, state, function(err) {
+								spf.end();
+								state.perf.count('literals', 1);
+								callback(err);
+							});
 						}
 						else {
-							self.searchWordIndex(task, state, callback);
+							// single word search
+							var spf = state.perf.begin('search_' + task.def.id + '_word');
+							self.searchWordIndex(task, state, function(err) {
+								spf.end();
+								state.perf.count('words', 1);
+								callback(err);
+							});
 						}
 					},
 					function(err) {
@@ -1347,6 +1385,7 @@ module.exports = Class.create({
 						if (err) {
 							self.logError('index', "Index search failed: " + err);
 							state.record_ids = {};
+							state.err = err;
 						}
 						self.logDebug(10, "Search complete", state.record_ids);
 						callback(null, state.record_ids, Tools.copyHashRemoveKeys(state, { config:1, record_ids:1, first:1 }));
@@ -1357,7 +1396,7 @@ module.exports = Class.create({
 	},
 	
 	searchWordIndex: function(query, state, callback) {
-		// run one search query (list of words against one index)
+		// run one word query (single word against one index)
 		var self = this;
 		var config = state.config;
 		var def = query.def;
@@ -1385,6 +1424,7 @@ module.exports = Class.create({
 		// then it would probably be faster to apply the logic using _data getMulti (a.k.a row scan).
 		// Otherwise, perform a normal hash merge (which has to read every hash page).
 		var hash_page_size = config.hash_page_size || 1000;
+		var wpf = state.perf.begin('word_' + query.word);
 		
 		if ((mode == 'and') && query.weight && (query.weight / hash_page_size > num_cur_items)) {
 			this.logDebug(10, "Performing row scan on " + num_cur_items + " items", query);
@@ -1394,7 +1434,9 @@ module.exports = Class.create({
 				return config.base_path + '/_data/' + record_id;
 			} );
 			
+			var rspf = state.perf.begin('row_scan');
 			this.getMulti( data_paths, function(err, datas) {
+				rspf.end();
 				if (err) return callback(err);
 				
 				datas.forEach( function(data, idx) {
@@ -1404,11 +1446,15 @@ module.exports = Class.create({
 					}
 				} );
 				
+				state.perf.count('rows_scanned', datas.length);
+				wpf.end();
 				callback();
 			} ); // getMulti
 		} // row scan
 		else {
 			this.logDebug(10, "Performing '" + mode + "' hash merge on " + num_cur_items + " items", query);
+			
+			var hmpf = state.perf.begin('hash_merge');
 			this.hashEachPage( path,
 				function(items, callback) {
 					switch (mode) {
@@ -1430,9 +1476,12 @@ module.exports = Class.create({
 							}
 						break;
 					}
+					state.perf.count('hash_pages', 1);
 					callback();
 				},
 				function(err) {
+					hmpf.end();
+					wpf.end();
 					if (mode == 'and') state.record_ids = new_items;
 					callback(err);
 				}
@@ -1460,6 +1509,7 @@ module.exports = Class.create({
 			function(word, callback) {
 				// for each word, iterate over record ids
 				var keepers = Object.create(null);
+				var wpf = state.perf.begin('literal_' + word);
 				
 				self.hashEachSync( path_prefix + word,
 					function(record_id, raw_value) {
@@ -1495,6 +1545,7 @@ module.exports = Class.create({
 						else keepers[record_id] = 1;
 					},
 					function(err) {
+						wpf.end();
 						// If in a subsequent word pass, make sure all temp_results
 						// ids are still matched in the latest word
 						if (temp_idx > 0) self.mergeIndex( temp_results, keepers, 'and' );

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.0.23",
+	"version": "3.1.0",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -21,17 +21,19 @@
 		"redis"
 	],
 	"dependencies": {
-		"pixl-server": "^1.0.0",
-		"pixl-class": "^1.0.3",
-		"pixl-tools": "^1.0.24",
-		"pixl-perf": "^1.0.0",
-		"pixl-cache": "^1.0.1",
+		"@aws-sdk/client-s3": "3.186.0",
 		"async": "2.6.4",
-		"nearley": "2.20.1",
+		"fast-stream-to-buffer": "1.0.0",
+		"he": "1.1.1",
 		"moo": "0.4.3",
+		"nearley": "2.20.1",
+		"pixl-cache": "^1.0.1",
+		"pixl-class": "^1.0.3",
+		"pixl-perf": "^1.0.0",
+		"pixl-server": "^1.0.0",
+		"pixl-tools": "^1.0.24",
 		"porter-stemmer": "0.9.1",
-		"unidecode": "0.1.8",
-		"he": "1.1.1"
+		"unidecode": "0.1.8"
 	},
 	"devDependencies": {
 		"pixl-unit": "^1.0.0"

package/test/test-main.js

@@ -446,10 +446,11 @@ module.exports = {
 				
 				self.storage.getStreamRange( key, 0, 5, function(err, storageStream, streamInfo) {
 					test.ok( !err, "No error fetching stream: " + err );
+					test.debug( "streamInfo: ", streamInfo );
 					test.ok( !!storageStream, "Got storage stream as 2nd arg");
 					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
 					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
-					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.len == 6, "Info has correct data length");
 					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
 					
 					outStream.on('finish', function() {
@@ -496,7 +497,7 @@ module.exports = {
 					test.ok( !!storageStream, "Got storage stream as 2nd arg");
 					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
 					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
-					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.len == 23, "Info has correct data length");
 					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
 					
 					outStream.on('finish', function() {
@@ -543,7 +544,7 @@ module.exports = {
 					test.ok( !!storageStream, "Got storage stream as 2nd arg");
 					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
 					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
-					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.len == 10, "Info has correct data length");
 					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
 					
 					outStream.on('finish', function() {