pixl-server-storage 3.1.3 → 3.1.6

package/engines/S3.js

@@ -7,6 +7,7 @@ var Component = require("pixl-server/component");
 var Tools = require("pixl-tools");
 var Cache = require("pixl-cache");
 var S3 = require("@aws-sdk/client-s3");
+var { NodeHttpHandler } = require("@aws-sdk/node-http-handler");
 var streamToBuffer = require("fast-stream-to-buffer");
 
 module.exports = Class.create({
@@ -83,6 +84,16 @@ module.exports = Class.create({
 		this.s3Params = combo_config.params || {};
 		delete combo_config.params;
 		
+		// allow user to specify HTTP timeout options for S3
+		if (combo_config.connectTimeout || combo_config.socketTimeout) {
+			combo_config.requestHandler = new NodeHttpHandler({
+				connectionTimeout: combo_config.connectTimeout || 0,
+				socketTimeout: combo_config.socketTimeout || 0
+			});
+			delete combo_config.connectTimeout;
+			delete combo_config.socketTimeout;
+		}
+		
 		this.s3 = new S3.S3Client(combo_config);
 	},
 	
@@ -408,9 +419,16 @@ module.exports = Class.create({
 					self.logDebug(9, "S3 stream download closed: " + key);
 				} );
 				
+				// get full length from the ContentRange header
+				var len = 0;
+				if (data.ContentRange && data.ContentRange.toString().match(/\/\s*(\d+)\s*$/)) {
+					len = parseInt( RegExp.$1 );
+				}
+				
 				callback( null, download, {
 					mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
-					len: data.ContentLength
+					len: len,
+					cr: data.ContentRange
 				} );
 			})
 			.catch( function(err) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.1.3",
+	"version": "3.1.6",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",

package/storage.js

@@ -35,7 +35,8 @@ module.exports = Class.create({
 		max_recent_events: 0,
 		cache_key_match: '',
 		expiration_updates: false,
-		lower_case_keys: true
+		lower_case_keys: true,
+		queue_timeout: 30000
 	},
 	
 	locks: null,
@@ -132,6 +133,7 @@ module.exports = Class.create({
 		this.maxRecentEvents = this.config.get('max_recent_events');
 		this.expHash = this.config.get('expiration_updates');
 		this.lowerKeys = this.config.get('lower_case_keys');
+		this.queueTimeout = this.config.get('queue_timeout');
 		
 		this.cacheKeyRegex = null;
 		if (this.config.get('cache_key_match')) {
@@ -550,7 +552,8 @@ module.exports = Class.create({
 			var func = task;
 			task = { action: 'custom', handler: func };
 		}
-		this.logDebug(9, "Enqueuing async task: " + (task.label || task.action), 
+		task._id = Tools.generateShortID();
+		this.logDebug(9, "Enqueuing async task: " + task._id + ": " + (task.label || task.action), 
 			this.debugLevel(10) ? task : null
 		);
 		this.queue.push( task );
@@ -559,10 +562,18 @@ module.exports = Class.create({
 	dequeue: function(task, callback) {
 		// run task and fire callback
 		var self = this;
-		this.logDebug(9, "Running async task: " + (task.label || task.action), 
+		this.logDebug(9, "Running async task: " + task._id + ": " + (task.label || task.action), 
 			this.debugLevel(10) ? task : null
 		);
 		
+		// optional timeout for queue item execution
+		var timer = this.queueTimeout ? timer = setTimeout( function() {
+			self.logError('queue', "Async task timed out: " + task._id + ": " + (task.label || task.action), { ms: self.queueTimeout });
+			if (callback) callback();
+			callback = null;
+			timer = null;
+		}, this.queueTimeout ) : null;
+		
 		switch (task.action) {
 			case 'expire_set':
 				// set expiration on record
@@ -585,7 +596,11 @@ module.exports = Class.create({
 								list_path: cleanup_list_path
 							});
 							
-							callback();
+							if (timer) clearTimeout(timer);
+							timer = null;
+							
+							if (callback) callback();
+							callback = null;
 						} ); // hashPut
 					} // expHash
 					else {
@@ -595,7 +610,11 @@ module.exports = Class.create({
 							list_path: cleanup_list_path
 						});
 						
-						callback();
+						if (timer) clearTimeout(timer);
+						timer = null;
+						
+						if (callback) callback();
+						callback = null;
 					}
 				} ); // listPush
 			break; // expire_set
@@ -604,7 +623,12 @@ module.exports = Class.create({
 				// custom handler
 				task.handler( task, function(err) {
 					if (err) self.logError('storage', "Failed to dequeue custom task: " + err);
-					callback();
+					
+					if (timer) clearTimeout(timer);
+					timer = null;
+					
+					if (callback) callback();
+					callback = null;
 				} );
 			break; // custom
 		} // switch action
@@ -995,6 +1019,10 @@ module.exports = Class.create({
 			last_second: this.lastSecondMetrics,
 			last_minute: this.lastMinuteMetrics,
 			recent_events: this.recentEvents,
+			queue: {
+				active: this.queue.running(),
+				pending: this.queue.length()
+			},
 			locks: {}
 		};