pixl-server-storage 3.0.4 → 3.0.8

package/engines/Couchbase.js

@@ -228,6 +228,32 @@ module.exports = Class.create({
 		} );
 	},
 	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		var self = this;
+		
+		// The Couchbase Node.JS 2.0 API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		this.get( key, function(err, buf) {
+			if (err && (err.code != CouchbaseAPI.errors.keyNotFound)) {
+				// some other error
+				err.message = "Failed to fetch key: " + key + ": " + err.message;
+				self.logError('couchbase', err.message);
+				return callback(err);
+			}
+			else if (!buf) {
+				// record not found
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback( err, null );
+			}
+			
+			var range = buf.slice(start, end + 1);
+			var stream = new BufferStream(range);
+			callback(null, stream);
+		} );
+	},
+	
 	delete: function(key, callback) {
 		// delete Couchbase key given key
 		// Example CB error message: The key does not exist on the server

package/engines/Filesystem.js

@@ -453,6 +453,36 @@ module.exports = Class.create({
 		} );
 	},
 	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and byte range
+		var self = this;
+		var file = this.getFilePath(key);
+		
+		this.logDebug(9, "Fetching ranged binary stream: " + key, { file, start, end } );
+		
+		// make sure record exists
+		fs.stat(file, function(err, stats) {
+			if (err) {
+				if (err.message.match(/ENOENT/)) {
+					err.message = "File not found";
+					err.code = "NoSuchKey";
+				}
+				else {
+					// log fs errors that aren't simple missing files (i.e. I/O errors)
+					self.logError('file', "Failed to stat file: " + key + ": " + file + ": " + err.message);
+				}
+				
+				err.message = "Failed to head key: " + key + ": " + err.message;
+				return callback( err, null );
+			}
+			
+			// create read stream
+			var inp = fs.createReadStream( file, { start, end } );
+			
+			callback( null, inp );
+		} );
+	},
+	
 	delete: function(key, callback) {
 		// delete key given key
 		var self = this;

package/engines/Hybrid.js

@@ -21,11 +21,13 @@ module.exports = Class.create({
 		// setup multiple sub-engines
 		this.logDebug(2, "Setting up hybrid engine", this.config.get() );
 		
-		this.binaryEngine = new require( './' + this.config.get('binaryEngine') + '.js' )();
+		var binaryClass = require( './' + this.config.get('binaryEngine') + '.js' );
+		this.binaryEngine = new binaryClass();
 		this.binaryEngine.storage = this.storage;
 		this.binaryEngine.init( this.server, this.storage.config.getSub( this.config.get('binaryEngine') ) );
 		
-		this.docEngine = new require( './' + this.config.get('docEngine') + '.js' )();
+		var docClass = require( './' + this.config.get('docEngine') + '.js' );
+		this.docEngine = new docClass();
 		this.docEngine.storage = this.storage;
 		this.docEngine.init( this.server, this.storage.config.getSub( this.config.get('docEngine') ) );
 		
@@ -77,6 +79,12 @@ module.exports = Class.create({
 		this.binaryEngine.getStream( key, callback );
 	},
 	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		// streams are binary only!
+		this.binaryEngine.getStreamRange( key, start, end, callback );
+	},
+	
 	delete: function(key, callback) {
 		// delete hybrid key given key
 		if (this.storage.isBinaryKey(key)) {

package/engines/Redis.js

@@ -231,6 +231,33 @@ module.exports = Class.create({
 		} );
 	},
 	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		var self = this;
+		
+		// The Redis API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		
+		this.get( key, function(err, buf) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('redis', ''+err);
+				return callback(err);
+			}
+			else if (!buf) {
+				// record not found
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback( err, null );
+			}
+			
+			var range = buf.slice(start, end + 1);
+			var stream = new BufferStream(range);
+			callback(null, stream);
+		} );
+	},
+	
 	delete: function(key, callback) {
 		// delete Redis key given key
 		var self = this;

package/engines/RedisCluster.js

@@ -248,6 +248,33 @@ module.exports = Class.create({
 		} );
 	},
 	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		var self = this;
+		
+		// The Redis API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		
+		this.get( key, function(err, buf) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('redis', ''+err);
+				return callback(err);
+			}
+			else if (!buf) {
+				// record not found
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback( err, null );
+			}
+			
+			var range = buf.slice(start, end + 1);
+			var stream = new BufferStream(range);
+			callback(null, stream);
+		} );
+	},
+	
 	delete: function(key, callback) {
 		// delete Redis key given key
 		var self = this;

package/engines/S3.js

@@ -125,7 +125,11 @@ module.exports = Class.create({
 			params.ContentType = 'application/json';
 		}
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.putObject( params, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				self.logError('s3', "Failed to store object: " + key + ": " + err.message);
 			}
@@ -152,7 +156,11 @@ module.exports = Class.create({
 		
 		this.logDebug(9, "Storing S3 Binary Stream: " + key);
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.upload(params, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				self.logError('s3', "Failed to store stream: " + key + ": " + err.message);
 			}
@@ -183,7 +191,11 @@ module.exports = Class.create({
 			return;
 		} // cache
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.headObject( { Key: this.extKey(key, orig_key) }, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				if (err.code != 'NoSuchKey') {
 					self.logError('s3', "Failed to head key: " + key + ": " + err.message);
@@ -227,7 +239,11 @@ module.exports = Class.create({
 			return;
 		} // cache
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.getObject( { Key: this.extKey(key, orig_key) }, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				if (err.code == 'NoSuchKey') {
 					// key not found, special case, don't log an error
@@ -279,7 +295,50 @@ module.exports = Class.create({
 		
 		var params = { Key: this.extKey(key, orig_key) };
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
+		this.s3.headObject( params, function(err) {
+			if (done) return; else done = true;
+			
+			if (err) {
+				if (err.code != 'NoSuchKey') {
+					self.logError('s3', "Failed to head key: " + key + ": " + err.message);
+				}
+				callback( err, null );
+				return;
+			}
+			
+			var download = self.s3.getObject(params).createReadStream();
+			
+			download.once('error', function(err) {
+				self.logError('s3', "Failed to download key: " + key + ": " + err.message);
+			});
+			download.once('end', function() {
+				self.logDebug(9, "S3 stream download complete: " + key);
+			} );
+			download.once('close', function() {
+				self.logDebug(9, "S3 stream download closed: " + key);
+			} );
+			
+			callback( null, download );
+		}); // headObject
+	},
+	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		var self = this;
+		var orig_key = key;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Fetching ranged S3 stream: " + key, { start, end });
+		
+		var params = { Key: this.extKey(key, orig_key) };
+		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.headObject( params, function(err) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				if (err.code != 'NoSuchKey') {
 					self.logError('s3', "Failed to head key: " + key + ": " + err.message);
@@ -288,15 +347,16 @@ module.exports = Class.create({
 				return;
 			}
 			
+			params.Range = "bytes=" + start + "-" + end;
 			var download = self.s3.getObject(params).createReadStream();
 			
-			download.on('error', function(err) {
+			download.once('error', function(err) {
 				self.logError('s3', "Failed to download key: " + key + ": " + err.message);
 			});
-			download.on('end', function() {
+			download.once('end', function() {
 				self.logDebug(9, "S3 stream download complete: " + key);
 			} );
-			download.on('close', function() {
+			download.once('close', function() {
 				self.logDebug(9, "S3 stream download closed: " + key);
 			} );
 			
@@ -312,7 +372,11 @@ module.exports = Class.create({
 		
 		this.logDebug(9, "Deleting S3 Object: " + key);
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.deleteObject( { Key: this.extKey(key, orig_key) }, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				self.logError('s3', "Failed to delete object: " + key + ": " + err.message);
 			}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.0.4",
+	"version": "3.0.8",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",

package/storage.js

@@ -354,6 +354,21 @@ module.exports = Class.create({
 		this.engine.getStream( key, callback );
 	},
 	
+	getStreamRange: function(key, start, end, callback) {
+		// fetch value via stream pipe and range
+		// start and end are both inclusive
+		var self = this;
+		if (!this.started) return callback( new Error("Storage has not completed startup.") );
+		
+		key = this.normalizeKey( key );
+		
+		if (!this.isBinaryKey(key)) {
+			return callback( new Error("Stream values are only allowed with keys containing file extensions, e.g. " + key + ".bin") );
+		}
+		
+		this.engine.getStreamRange( key, start, end, callback );
+	},
+	
 	getMulti: function(keys, callback) {
 		// fetch multiple records at once, given array of keys
 		// callback is provided an array of values in matching order to keys

package/transaction.js

@@ -27,14 +27,6 @@ var TransStorageFunctions = {
 		this.tempFileCounter = 1;
 	},
 	
-	getFilePath: function(key) {
-		// get local path to file given storage key
-		var trans = this.transactions[ this.currentTransactionPath ];
-		var key_id = Tools.digestHex( key );
-		var file = Path.join( this.transDir, "data", trans.id + '-' + key_id + '.json' );
-		return file;
-	},
-	
 	put: function(key, value, callback) {
 		// store key+value pair in transaction
 		var self = this;
@@ -645,35 +637,16 @@ module.exports = Class.create({
 						self.logError('rollback', "Unable to delete rollback log: " + trans.log + ": " + err);
 					}
 					
-					// also delete any files written to temp area
-					if (!trans.keys) trans.keys = {};
+					// complete, unlock and remove transaction from memory
+					self.unlock( 'C|'+path );
+					self._transUnlock(path);
+					self.transactions[path].keys = {}; // release memory
+					self.transactions[path].values = {}; // release memory
+					self.transactions[path].queue = []; // release memory
+					delete self.transactions[path];
 					
-					async.forEachOfLimit( trans.keys, self.concurrency, 
-						function(value, key, callback) {
-							// delete temp key file
-							var key_id = Tools.digestHex( key );
-							var file = Path.join( self.transDir, "data", trans.id + '-' + key_id + '.json' );
-							
-							fs.unlink( file, function(err) {
-								if (err && !err.message.match(/ENOENT/)) {
-									self.logError('rollback', "Unable to delete transaction file: " + file + ": " + err);
-								}
-								callback();
-							});
-						},
-						function(err) {
-							// complete, unlock and remove transaction from memory
-							self.unlock( 'C|'+path );
-							self._transUnlock(path);
-							self.transactions[path].keys = {}; // release memory
-							self.transactions[path].values = {}; // release memory
-							self.transactions[path].queue = []; // release memory
-							delete self.transactions[path];
-							
-							self.logDebug(3, "Transaction rollback complete: " + trans.id, { path: path });
-							callback();
-						}
-					); // forEachOfLimit
+					self.logDebug(3, "Transaction rollback complete: " + trans.id, { path: path });
+					callback();
 				}); // fs.unlink
 			} // done with log
 		); // fileEachLine