pixl-server-storage 3.1.14 → 3.1.16

package/engines/S3.js

@@ -10,6 +10,7 @@ var S3 = require("@aws-sdk/client-s3");
 var { Upload } = require("@aws-sdk/lib-storage");
 var { NodeHttpHandler } = require("@smithy/node-http-handler");
 var streamToBuffer = require("fast-stream-to-buffer");
+var StreamMeter = require("stream-meter");
 
 module.exports = Class.create({
 	
@@ -32,14 +33,15 @@ module.exports = Class.create({
 		var aws_config = this.storage.config.get('AWS') || this.server.config.get('AWS');
 		var s3_config = this.config.get();
 		
-		this.logDebug(2, "Setting up Amazon S3 (" + aws_config.region + ")");
-		this.logDebug(3, "S3 Bucket ID: " + s3_config.params.Bucket);
+		this.logDebug(5, "Setting up Amazon S3 (" + aws_config.region + ")");
+		this.logDebug(6, "S3 Bucket ID: " + s3_config.params.Bucket);
 		
 		this.keyPrefix = (s3_config.keyPrefix || '').replace(/^\//, '');
 		if (this.keyPrefix && !this.keyPrefix.match(/\/$/)) this.keyPrefix += '/';
 		
 		this.keyTemplate = (s3_config.keyTemplate || '').replace(/^\//, '').replace(/\/$/, '');
 		this.fileExtensions = !!s3_config.fileExtensions;
+		this.pretty = !!s3_config.pretty;
 		
 		// optional LRU cache
 		this.cache = null;
@@ -80,6 +82,7 @@ module.exports = Class.create({
 		delete combo_config.keyPrefix;
 		delete combo_config.keyTemplate;
 		delete combo_config.fileExtensions;
+		delete combo_config.pretty;
 		delete combo_config.cache;
 		
 		this.s3Params = combo_config.params || {};
@@ -100,8 +103,6 @@ module.exports = Class.create({
 	
 	prepKey: function(key) {
 		// prepare key for S3 based on config
-		var md5 = Tools.digestHex(key, 'md5');
-		
 		var ns = '';
 		if (key.match(/^([\w\-\.]+)\//)) ns = RegExp.$1;
 		
@@ -110,6 +111,7 @@ module.exports = Class.create({
 		}
 		
 		if (this.keyTemplate) {
+			var md5 = Tools.digestHex(key, 'md5');
 			var idx = 0;
 			var temp = this.keyTemplate.replace( /\#/g, function() {
 				return md5.substr(idx++, 1);
@@ -146,13 +148,14 @@ module.exports = Class.create({
 		}
 		else {
 			this.logDebug(9, "Storing S3 JSON Object: " + key, this.debugLevel(10) ? params.Body : null);
-			params.Body = JSON.stringify( params.Body );
+			params.Body = this.pretty ? JSON.stringify( params.Body, null, "\t" ) : JSON.stringify( params.Body );
 			params.ContentType = 'application/json';
 		}
 		
 		this.s3.send( new S3.PutObjectCommand(params) )
 			.then( function(data) {
 				self.logDebug(9, "Store complete: " + key);
+				self.storage.emit('billing', 's3_put', 1);
 				
 				// possibly cache in LRU
 				if (self.cache && !is_binary) {
@@ -162,6 +165,13 @@ module.exports = Class.create({
 				if (callback) process.nextTick( function() { callback(null, data); });
 			} )
 			.catch( function(err) {
+				if (err.name == 'SlowDown') {
+					// special behavior for SlowDown errors
+					self.logDebug(6, "Received SlowDown from S3 put: " + orig_key + ": " + err + " (will retry)");
+					self.storage.emit('slowDown');
+					setTimeout( function() { self.put(orig_key, value, callback); }, 1000 );
+					return;
+				}
 				self.logError('s3', "Failed to store object: " + key + ": " + (err.message || err), err);
 				if (callback) process.nextTick( function() { callback(err); });
 			} );
@@ -173,9 +183,12 @@ module.exports = Class.create({
 		var orig_key = key;
 		key = this.prepKey(key);
 		
+		var meter = new StreamMeter();
+		inp.pipe(meter);
+		
 		var params = Tools.copyHash( this.s3Params );
 		params.Key = this.extKey(key, orig_key);
-		params.Body = inp;
+		params.Body = meter;
 		
 		this.logDebug(9, "Storing S3 Binary Stream: " + key);
 		
@@ -187,6 +200,8 @@ module.exports = Class.create({
 		upload.done()
 			.then( function(data) {
 				self.logDebug(9, "Stream store complete: " + key);
+				self.storage.emit('billing', 's3_put', 1);
+				self.storage.emit('billing', 's3_bytes_out', meter.bytes);
 				if (callback) process.nextTick( function() { callback(null, data); });
 			} )
 			.catch( function(err) {
@@ -201,9 +216,12 @@ module.exports = Class.create({
 		var orig_key = key;
 		key = this.prepKey(key);
 		
+		var meter = new StreamMeter();
+		inp.pipe(meter);
+		
 		var params = Tools.copyHash( this.s3Params );
 		params.Key = this.extKey(key, orig_key);
-		params.Body = inp;
+		params.Body = meter;
 		if (opts) Tools.mergeHashInto(params, opts);
 		
 		this.logDebug(9, "Storing S3 Binary Stream: " + key);
@@ -216,6 +234,8 @@ module.exports = Class.create({
 		upload.done()
 			.then( function(data) {
 				self.logDebug(9, "Stream store complete: " + key);
+				self.storage.emit('billing', 's3_put', 1);
+				self.storage.emit('billing', 's3_bytes_out', meter.bytes);
 				if (callback) process.nextTick( function() { callback(null, data); });
 			} )
 			.catch( function(err) {
@@ -252,6 +272,7 @@ module.exports = Class.create({
 		this.s3.send( new S3.HeadObjectCommand(params) )
 			.then( function(data) {
 				self.logDebug(9, "Head complete: " + key);
+				self.storage.emit('billing', 's3_head', 1);
 				
 				process.nextTick( function() {
 					callback( null, {
@@ -267,6 +288,13 @@ module.exports = Class.create({
 					err = new Error("Failed to head key: " + key + ": Not found");
 					err.code = "NoSuchKey";
 				}
+				else if (err.name == 'SlowDown') {
+					// special behavior for SlowDown errors
+					self.logDebug(6, "Received SlowDown from S3 head: " + orig_key + ": " + err + " (will retry)");
+					self.storage.emit('slowDown');
+					setTimeout( function() { self.head(orig_key, callback); }, 1000 );
+					return;
+				}
 				else {
 					// some other error
 					self.logError('s3', "Failed to head key: " + key + ": " + (err.message || err), err);
@@ -314,6 +342,9 @@ module.exports = Class.create({
 						return callback(err);
 					}
 					
+					self.storage.emit('billing', 's3_get', 1);
+					self.storage.emit('billing', 's3_bytes_in', body.length);
+					
 					if (is_binary) {
 						self.logDebug(9, "Binary fetch complete: " + key, '' + body.length + ' bytes');
 					}
@@ -347,6 +378,13 @@ module.exports = Class.create({
 					err = new Error("Failed to fetch key: " + key + ": Not found");
 					err.code = "NoSuchKey";
 				}
+				else if (err.name == 'SlowDown') {
+					// special behavior for SlowDown errors
+					self.logDebug(6, "Received SlowDown from S3 get: " + orig_key + ": " + err + " (will retry)");
+					self.storage.emit('slowDown');
+					setTimeout( function() { self.get(orig_key, callback); }, 1000 );
+					return;
+				}
 				else {
 					// some other error
 					self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
@@ -376,6 +414,8 @@ module.exports = Class.create({
 					}
 					
 					self.logDebug(9, "Binary fetch complete: " + key, '' + body.length + ' bytes');
+					self.storage.emit('billing', 's3_get', 1);
+					self.storage.emit('billing', 's3_bytes_in', body.length);
 					
 					callback( null, body, {
 						mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
@@ -390,6 +430,13 @@ module.exports = Class.create({
 					err = new Error("Failed to fetch key: " + key + ": Not found");
 					err.code = "NoSuchKey";
 				}
+				else if (err.name == 'SlowDown') {
+					// special behavior for SlowDown errors
+					self.logDebug(6, "Received SlowDown from S3 getBuffer: " + orig_key + ": " + err + " (will retry)");
+					self.storage.emit('slowDown');
+					setTimeout( function() { self.getBuffer(orig_key, callback); }, 1000 );
+					return;
+				}
 				else {
 					// some other error
 					self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
@@ -423,6 +470,9 @@ module.exports = Class.create({
 					self.logDebug(9, "S3 stream download closed: " + key);
 				} );
 				
+				self.storage.emit('billing', 's3_get', 1);
+				self.storage.emit('billing', 's3_bytes_in', data.ContentLength);
+				
 				process.nextTick( function() {
 					callback( null, download, {
 						mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
@@ -478,6 +528,9 @@ module.exports = Class.create({
 					self.logDebug(9, "S3 stream download closed: " + key);
 				} );
 				
+				self.storage.emit('billing', 's3_get', 1);
+				self.storage.emit('billing', 's3_bytes_in', data.ContentRange);
+				
 				// get full length from the ContentRange header
 				var len = 0;
 				if (data.ContentRange && data.ContentRange.toString().match(/\/\s*(\d+)\s*$/)) {
@@ -521,6 +574,7 @@ module.exports = Class.create({
 		this.s3.send( new S3.DeleteObjectCommand(params) )
 			.then( function(data) {
 				self.logDebug(9, "Delete complete: " + key);
+				self.storage.emit('billing', 's3_delete', 1);
 				
 				// possibly delete from LRU cache as well
 				if (self.cache && self.cache.has(orig_key)) {
@@ -530,6 +584,13 @@ module.exports = Class.create({
 				if (callback) process.nextTick( function() { callback(null, data); } );
 			} )
 			.catch( function(err) {
+				if (err.name == 'SlowDown') {
+					// special behavior for SlowDown errors
+					self.logDebug(6, "Received SlowDown from S3 delete: " + orig_key + ": " + err + " (will retry)");
+					self.storage.emit('slowDown');
+					setTimeout( function() { self.delete(orig_key, callback); }, 1000 );
+					return;
+				}
 				self.logError('s3', "Failed to delete object: " + key + ": " + (err.message || err), err);
 				if (callback) process.nextTick( function() { callback(err); } );
 			} );

package/indexer.js

@@ -75,7 +75,7 @@ module.exports = Class.create({
 	validateIndexConfig: function(config) {
 		// make sure index config is kosher
 		// return false for success, or error on failure
-		if (!config || !config.fields || !config.fields.length) {
+		if (!config || !config.fields || !Tools.isaArray(config.fields)) {
 			return( new Error("Invalid index configuration object.") );
 		}
 		if (Tools.findObject(config.fields, { _primary: 1 })) {
@@ -104,6 +104,10 @@ module.exports = Class.create({
 		
 		// validate each sorter def
 		if (config.sorters) {
+			if (!Tools.isaArray(config.sorters)) {
+				return( new Error("Invalid index sorters array.") );
+			}
+			
 			for (var idx = 0, len = config.sorters.length; idx < len; idx++) {
 				var sorter = config.sorters[idx];
 				
@@ -163,18 +167,12 @@ module.exports = Class.create({
 		var fields = [];
 		
 		config.fields.forEach( function(def) {
-			var value = def.source.match(/^\//) ? Tools.lookupPath(def.source, record) : Tools.sub(def.source, record, true);
+			var value = def.source.match(/\[.+\]/) ? Tools.sub(def.source, record, true) : Tools.getPath(record, def.source);
+			if (value === undefined) value = null;
 			if ((value === null) && ("default_value" in def)) value = def.default_value;
 			if (value !== null) fields.push(def);
 		} );
 		
-		if (!fields.length) {
-			// nothing to index!
-			this.logDebug(6, "Nothing to index, skipping entire record");
-			if (callback) callback();
-			return;
-		}
-		
 		// start index and track perf
 		var pf = this.perf.begin('index');
 		
@@ -211,7 +209,8 @@ module.exports = Class.create({
 							return;
 						}
 						
-						var value = def.source.match(/^\//) ? Tools.lookupPath(def.source, record) : Tools.sub(def.source, record, true);
+						var value = def.source.match(/\[.+\]/) ? Tools.sub(def.source, record, true) : Tools.getPath(record, def.source);
+						if (value === undefined) value = null;
 						if ((value === null) && ("default_value" in def)) value = def.default_value;
 						if (typeof(value) == 'object') value = JSON.stringify(value);
 						
@@ -920,7 +919,8 @@ module.exports = Class.create({
 		// add record to sorter index
 		var config = state.config;
 		
-		var value = Tools.lookupPath(sorter.source, record);
+		var value = Tools.getPath(record, sorter.source);
+		if (value === undefined) value = null;
 		if ((value === null) && ("default_value" in sorter)) value = sorter.default_value;
 		if (value === null) {
 			if (state.new_record) value = ((sorter.type == 'number') ? 0 : '');
@@ -975,6 +975,11 @@ module.exports = Class.create({
 		return value.replace(/\W+/g, '_').replace(/_+/g, '_');
 	},
 	
+	filterWords_alphanum_array: function(value) {
+		// filter out everything except alphanum + underscore + comma, suitable for JSON arrays
+		return value.replace(/[\[\]\"\']+/g, '').replace(/[^\w\,]+/g, '_').replace(/_+/g, '_');
+	},
+	
 	getWordList: function(value, def, config) {
 		// clean and filter text down to list of alphanumeric words
 		// return array of clean words

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.1.14",
+	"version": "3.1.16",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -36,6 +36,7 @@
 		"pixl-server": "^1.0.39",
 		"pixl-tools": "^1.0.33",
 		"porter-stemmer": "0.9.1",
+		"stream-meter": "1.0.4",
 		"unidecode": "0.1.8"
 	},
 	"devDependencies": {

package/storage.js

@@ -56,7 +56,7 @@ module.exports = Class.create({
 	startup: function(callback) {
 		// setup storage plugin
 		var self = this;
-		this.logDebug(2, "Setting up storage system v" + this.version);
+		this.logDebug(5, "Setting up storage system v" + this.version);
 		
 		// advisory locking system (in RAM, single process only)
 		this.locks = {};