pixl-server-storage 3.0.5 → 3.0.9

package/README.md

@@ -846,7 +846,7 @@ storage.getStream( 'test1.gif', function(err, readStream) {
 } );
 ```
 
-Please note that not all the storage engines support streams natively, so the content may actually be loaded into RAM in the background.  Namely, as of this writing, the Couchbase and Redis APIs does not support streams, so they are currently simulated in those engines.  Streams *are* supported natively in both the Filesystem and Amazon S3 engines.
+Please note that not all the storage engines support streams natively, so the content may actually be loaded into RAM in the background.  Namely, as of this writing, the Couchbase and Redis APIs do not support streams, so they are currently simulated in those engines.  Streams *are* supported natively in both the Filesystem and Amazon S3 engines.
 
 # Expiring Data
 

package/docs/API.md

@@ -18,6 +18,7 @@ var storage = server.Storage;
 	* [get](#get)
 	* [getMulti](#getmulti)
 	* [getStream](#getstream)
+	* [getStreamRange](#getstreamrange)
 	* [head](#head)
 	* [headMulti](#headmulti)
 	* [delete](#delete)
@@ -126,6 +127,8 @@ storage.putMulti( records, function(err) {
 } );
 ```
 
+Note that if any of the individual put operations fail, the entire `putMulti()` function is aborted, and the first error is passed to your callback.  At this point the operation may have been partially successful, with some records written, and others not.  Due to this uncertainty, you may want to use this method inside of a [transaction](./Transactions.md), which can be safely rolled back upon error.
+
 ## putStream
 
 ```javascript
@@ -142,6 +145,8 @@ storage.putStream( 'test1.gif', stream, function(err) {
 } );
 ```
 
+Please note that as of this writing, the `Couchbase`, `Redis` and `RedisCluster` engines have no native stream API, so the `putStream()` method has to load the entire record into memory.
+
 ## get
 
 ```javascript
@@ -173,19 +178,46 @@ storage.getMulti( ['test1', 'test2', 'test3'], function(err, values) {
 } );
 ```
 
+Note that if *any* of the records fail, the entire operation fails, and the first error is passed to your callback.
+
 ## getStream
 
 ```javascript
 storage.getStream( KEY, CALLBACK );
 ```
 
-The `getStream()` method retrieves a [readable stream](https://nodejs.org/api/stream.html#stream_class_stream_readable) to a given record's data, so it can be read or piped to a writable stream.  This is for very large records, so nothing is loaded into memory.  Example of spooling to a local file:
+The `getStream()` method retrieves a [readable stream](https://nodejs.org/api/stream.html#class-streamreadable) to a given record's data, so it can be read or piped to a writable stream.  This is for very large records, so nothing is loaded into memory.  Example of spooling to a local file:
+
+```javascript
+var fs = require('fs');
+var writeStream = fs.createWriteStream('/var/tmp/downloaded.gif');
+
+storage.getStream( 'test1.gif', function(err, readStream, info) {
+	if (err) throw err;
+	writeStream.on('finish', function() {
+		// data is completely written
+	} );
+	readStream.pipe( writeStream );
+} );
+```
+
+As you can see above, your callback is also passed a 3rd argument, which is an object containing the record's full byte length and modification date.  These properties match those returned when you call [head()](#head) (i.e. `len` and `mod`).
+
+Please note that as of this writing, the `Couchbase`, `Redis` and `RedisCluster` engines have no native stream API, so the `getStream()` method has to load the entire record into memory.
+
+## getStreamRange
+
+```javascript
+storage.getStreamRange( KEY, START, END, CALLBACK );
+```
+
+The `getStreamRange()` method retrieves a [readable stream](https://nodejs.org/api/stream.html#class-streamreadable) to a specific slice of a record's data, so it can be read or piped to a writable stream.  The `start` and `end` arguments should be set to the starting and ending byte offset of the slice you want.  Both values are *inclusive*.  Example of spooling bytes `0-99` of a record to a local file:
 
 ```javascript
 var fs = require('fs');
 var writeStream = fs.createWriteStream('/var/tmp/downloaded.gif');
 
-storage.getStream( 'test1.gif', function(err, readStream) {
+storage.getStreamRange( 'test1.gif', 0, 99, function(err, readStream, info) {
 	if (err) throw err;
 	writeStream.on('finish', function() {
 		// data is completely written
@@ -194,6 +226,10 @@ storage.getStream( 'test1.gif', function(err, readStream) {
 } );
 ```
 
+As you can see above, your callback is also passed a 3rd argument, which is an object containing the record's full byte length and modification date.  These properties match those returned when you call [head()](#head) (i.e. `len` and `mod`).  The `len` is not affected by the `start` and `end` range -- it is always the full byte length of the record.
+
+Please note that as of this writing, the `Couchbase`, `Redis` and `RedisCluster` engines have no native stream API, so the `getStreamRange()` method has to load the entire record into memory.
+
 ## head
 
 ```javascript
@@ -217,7 +253,7 @@ storage.head( 'test1', function(err, data) {
 } );
 ```
 
-Please note that as of this writing, the `Couchbase` engine has no native API, so the `head()` method has to load the entire record.  It does return the record size in to the `len` property, but there is no way to retrieve the last modified date.
+Please note that as of this writing, the `Couchbase`, `Redis` and `RedisCluster` engines have no native head API, so the `head()` method has to load the entire record.  It does return the record size in to the `len` property, but there is no way to retrieve the last modified date.
 
 ## headMulti
 

package/engines/Couchbase.js

@@ -224,7 +224,33 @@ module.exports = Class.create({
 			}
 			
 			var stream = new BufferStream(buf);
-			callback(null, stream);
+			callback(null, stream, { mod: 1, len: buf.length });
+		} );
+	},
+	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		var self = this;
+		
+		// The Couchbase Node.JS 2.0 API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		this.get( key, function(err, buf) {
+			if (err && (err.code != CouchbaseAPI.errors.keyNotFound)) {
+				// some other error
+				err.message = "Failed to fetch key: " + key + ": " + err.message;
+				self.logError('couchbase', err.message);
+				return callback(err);
+			}
+			else if (!buf) {
+				// record not found
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback( err, null );
+			}
+			
+			var range = buf.slice(start, end + 1);
+			var stream = new BufferStream(range);
+			callback(null, stream, { mod: 1, len: buf.length });
 		} );
 	},
 	

package/engines/Filesystem.js

@@ -449,7 +449,43 @@ module.exports = Class.create({
 			// create read stream
 			var inp = fs.createReadStream( file );
 			
-			callback( null, inp );
+			callback( null, inp, {
+				mod: Math.floor(stats.mtime.getTime() / 1000),
+				len: stats.size
+			} );
+		} );
+	},
+	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and byte range
+		var self = this;
+		var file = this.getFilePath(key);
+		
+		this.logDebug(9, "Fetching ranged binary stream: " + key, { file, start, end } );
+		
+		// make sure record exists
+		fs.stat(file, function(err, stats) {
+			if (err) {
+				if (err.message.match(/ENOENT/)) {
+					err.message = "File not found";
+					err.code = "NoSuchKey";
+				}
+				else {
+					// log fs errors that aren't simple missing files (i.e. I/O errors)
+					self.logError('file', "Failed to stat file: " + key + ": " + file + ": " + err.message);
+				}
+				
+				err.message = "Failed to head key: " + key + ": " + err.message;
+				return callback( err, null );
+			}
+			
+			// create read stream
+			var inp = fs.createReadStream( file, { start, end } );
+			
+			callback( null, inp, {
+				mod: Math.floor(stats.mtime.getTime() / 1000),
+				len: stats.size
+			} );
 		} );
 	},
 	

package/engines/Hybrid.js

@@ -79,6 +79,12 @@ module.exports = Class.create({
 		this.binaryEngine.getStream( key, callback );
 	},
 	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		// streams are binary only!
+		this.binaryEngine.getStreamRange( key, start, end, callback );
+	},
+	
 	delete: function(key, callback) {
 		// delete hybrid key given key
 		if (this.storage.isBinaryKey(key)) {

package/engines/Redis.js

@@ -227,7 +227,34 @@ module.exports = Class.create({
 			}
 			
 			var stream = new BufferStream(buf);
-			callback(null, stream);
+			callback(null, stream, { mod: 1, len: buf.length });
+		} );
+	},
+	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		var self = this;
+		
+		// The Redis API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		
+		this.get( key, function(err, buf) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('redis', ''+err);
+				return callback(err);
+			}
+			else if (!buf) {
+				// record not found
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback( err, null );
+			}
+			
+			var range = buf.slice(start, end + 1);
+			var stream = new BufferStream(range);
+			callback(null, stream, { mod: 1, len: buf.length });
 		} );
 	},
 	

package/engines/RedisCluster.js

@@ -244,7 +244,34 @@ module.exports = Class.create({
 			}
 			
 			var stream = new BufferStream(buf);
-			callback(null, stream);
+			callback(null, stream, { mod: 1, len: buf.length });
+		} );
+	},
+	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		var self = this;
+		
+		// The Redis API has no stream support.
+		// So, we have to do this the RAM-hard way...
+		
+		this.get( key, function(err, buf) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('redis', ''+err);
+				return callback(err);
+			}
+			else if (!buf) {
+				// record not found
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				return callback( err, null );
+			}
+			
+			var range = buf.slice(start, end + 1);
+			var stream = new BufferStream(range);
+			callback(null, stream, { mod: 1, len: buf.length });
 		} );
 	},
 	

package/engines/S3.js

@@ -125,7 +125,11 @@ module.exports = Class.create({
 			params.ContentType = 'application/json';
 		}
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.putObject( params, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				self.logError('s3', "Failed to store object: " + key + ": " + err.message);
 			}
@@ -152,7 +156,11 @@ module.exports = Class.create({
 		
 		this.logDebug(9, "Storing S3 Binary Stream: " + key);
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.upload(params, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				self.logError('s3', "Failed to store stream: " + key + ": " + err.message);
 			}
@@ -183,7 +191,11 @@ module.exports = Class.create({
 			return;
 		} // cache
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.headObject( { Key: this.extKey(key, orig_key) }, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				if (err.code != 'NoSuchKey') {
 					self.logError('s3', "Failed to head key: " + key + ": " + err.message);
@@ -227,7 +239,11 @@ module.exports = Class.create({
 			return;
 		} // cache
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.getObject( { Key: this.extKey(key, orig_key) }, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				if (err.code == 'NoSuchKey') {
 					// key not found, special case, don't log an error
@@ -278,29 +294,82 @@ module.exports = Class.create({
 		this.logDebug(9, "Fetching S3 Stream: " + key);
 		
 		var params = { Key: this.extKey(key, orig_key) };
+		var download = self.s3.getObject(params).createReadStream();
+		var proceed = false;
+		
+		download.on('error', function(err) {
+			if (proceed) self.logError('s3', "Failed to download key: " + key + ": " + err.message);
+		});
+		download.once('end', function() {
+			self.logDebug(9, "S3 stream download complete: " + key);
+		} );
+		download.once('close', function() {
+			self.logDebug(9, "S3 stream download closed: " + key);
+		} );
 		
-		this.s3.headObject( params, function(err) {
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
+		this.s3.headObject( params, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				if (err.code != 'NoSuchKey') {
 					self.logError('s3', "Failed to head key: " + key + ": " + err.message);
 				}
+				download.destroy();
 				callback( err, null );
 				return;
 			}
 			
-			var download = self.s3.getObject(params).createReadStream();
-			
-			download.on('error', function(err) {
-				self.logError('s3', "Failed to download key: " + key + ": " + err.message);
-			});
-			download.on('end', function() {
-				self.logDebug(9, "S3 stream download complete: " + key);
-			} );
-			download.on('close', function() {
-				self.logDebug(9, "S3 stream download closed: " + key);
+			proceed = true;
+			callback( null, download, {
+				mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
+				len: data.ContentLength
 			} );
+		}); // headObject
+	},
+	
+	getStreamRange: function(key, start, end, callback) {
+		// get readable stream to record value given key and range
+		var self = this;
+		var orig_key = key;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Fetching ranged S3 stream: " + key, { start, end });
+		
+		var params = { Key: this.extKey(key, orig_key) };
+		var download = self.s3.getObject( Tools.mergeHashes(params, { Range: "bytes=" + start + "-" + end }) ).createReadStream();
+		var proceed = false;
+		
+		download.on('error', function(err) {
+			if (proceed) self.logError('s3', "Failed to download key: " + key + ": " + err.message);
+		});
+		download.once('end', function() {
+			self.logDebug(9, "S3 stream download complete: " + key);
+		} );
+		download.once('close', function() {
+			self.logDebug(9, "S3 stream download closed: " + key);
+		} );
+		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
+		this.s3.headObject( params, function(err, data) {
+			if (done) return; else done = true;
 			
-			callback( null, download );
+			if (err) {
+				if (err.code != 'NoSuchKey') {
+					self.logError('s3', "Failed to head key: " + key + ": " + err.message);
+				}
+				download.destroy();
+				callback( err, null );
+				return;
+			}
+			
+			proceed = true;
+			callback( null, download, {
+				mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
+				len: data.ContentLength
+			} );
 		}); // headObject
 	},
 	
@@ -312,7 +381,11 @@ module.exports = Class.create({
 		
 		this.logDebug(9, "Deleting S3 Object: " + key);
 		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
 		this.s3.deleteObject( { Key: this.extKey(key, orig_key) }, function(err, data) {
+			if (done) return; else done = true;
+			
 			if (err) {
 				self.logError('s3', "Failed to delete object: " + key + ": " + err.message);
 			}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.0.5",
+	"version": "3.0.9",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",

package/storage.js

@@ -354,6 +354,21 @@ module.exports = Class.create({
 		this.engine.getStream( key, callback );
 	},
 	
+	getStreamRange: function(key, start, end, callback) {
+		// fetch value via stream pipe and range
+		// start and end are both inclusive
+		var self = this;
+		if (!this.started) return callback( new Error("Storage has not completed startup.") );
+		
+		key = this.normalizeKey( key );
+		
+		if (!this.isBinaryKey(key)) {
+			return callback( new Error("Stream values are only allowed with keys containing file extensions, e.g. " + key + ".bin") );
+		}
+		
+		this.engine.getStreamRange( key, start, end, callback );
+	},
+	
 	getMulti: function(keys, callback) {
 		// fetch multiple records at once, given array of keys
 		// callback is provided an array of values in matching order to keys

package/test/test-main.js

@@ -375,7 +375,7 @@ module.exports = {
 		},
 		
 		function testStream(test) {
-			test.expect(11);
+			test.expect(14);
 			var self = this;
 			
 			var key = 'spacer-stream.gif';
@@ -395,10 +395,13 @@ module.exports = {
 				var tempFile = __dirname + '/' + filename + '.streamtemp';
 				var outStream = fs.createWriteStream( tempFile );
 				
-				self.storage.getStream( key, function(err, storageStream) {
+				self.storage.getStream( key, function(err, storageStream, streamInfo) {
 					test.ok( !err, "No error fetching stream: " + err );
 					test.ok( !!storageStream, "Got storage stream as 2nd arg");
 					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
+					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
+					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
 					
 					outStream.on('finish', function() {
 						var newSpacerBuf = fs.readFileSync( tempFile );
@@ -420,6 +423,52 @@ module.exports = {
 			} ); // putStream
 		},
 		
+		function testStreamRange(test) {
+			test.expect(14);
+			var self = this;
+			
+			var key = 'spacer-stream.gif';
+			var filename = 'spacer.gif';
+			var spacerBuf = fs.readFileSync( __dirname + '/' + filename );
+			var spacerStream = fs.createReadStream( __dirname + '/' + filename );
+			
+			test.ok( !!spacerBuf, "Got buffer from file" );
+			test.ok( typeof(spacerBuf) == 'object', "Buffer is an object" );
+			test.ok( spacerBuf.length > 0, "Buffer has size" );
+			test.ok( !!spacerStream, "Got read stream" );
+			
+			this.storage.putStream( key, spacerStream, function(err) {
+				test.ok( !err, "No error creating stream: " + err );
+				
+				var tempFile = __dirname + '/' + filename + '.streamtemp';
+				var outStream = fs.createWriteStream( tempFile );
+				
+				self.storage.getStreamRange( key, 0, 5, function(err, storageStream, streamInfo) {
+					test.ok( !err, "No error fetching stream: " + err );
+					test.ok( !!storageStream, "Got storage stream as 2nd arg");
+					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
+					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
+					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
+					
+					outStream.on('finish', function() {
+						var newSpacerBuf = fs.readFileSync( tempFile );
+						test.ok( newSpacerBuf.length == 6, "Stream length is correct" );
+						test.ok( newSpacerBuf.toString() == "GIF89a", "Range buffer content is correct" );
+						
+						self.storage.delete( key, function(err) {
+							test.ok( !err, "No error deleting stream key: " + err );
+							fs.unlinkSync( tempFile );
+							test.done();
+						} ); // delete
+					} ); // stream finish
+					
+					storageStream.pipe( outStream );
+					
+				} ); // getStream
+			} ); // putStream
+		},
+		
 		function testPutMulti(test) {
 			// test storing multiple keys at once
 			test.expect(1);

package/transaction.js

@@ -27,14 +27,6 @@ var TransStorageFunctions = {
 		this.tempFileCounter = 1;
 	},
 	
-	getFilePath: function(key) {
-		// get local path to file given storage key
-		var trans = this.transactions[ this.currentTransactionPath ];
-		var key_id = Tools.digestHex( key );
-		var file = Path.join( this.transDir, "data", trans.id + '-' + key_id + '.json' );
-		return file;
-	},
-	
 	put: function(key, value, callback) {
 		// store key+value pair in transaction
 		var self = this;
@@ -645,35 +637,16 @@ module.exports = Class.create({
 						self.logError('rollback', "Unable to delete rollback log: " + trans.log + ": " + err);
 					}
 					
-					// also delete any files written to temp area
-					if (!trans.keys) trans.keys = {};
+					// complete, unlock and remove transaction from memory
+					self.unlock( 'C|'+path );
+					self._transUnlock(path);
+					self.transactions[path].keys = {}; // release memory
+					self.transactions[path].values = {}; // release memory
+					self.transactions[path].queue = []; // release memory
+					delete self.transactions[path];
 					
-					async.forEachOfLimit( trans.keys, self.concurrency, 
-						function(value, key, callback) {
-							// delete temp key file
-							var key_id = Tools.digestHex( key );
-							var file = Path.join( self.transDir, "data", trans.id + '-' + key_id + '.json' );
-							
-							fs.unlink( file, function(err) {
-								if (err && !err.message.match(/ENOENT/)) {
-									self.logError('rollback', "Unable to delete transaction file: " + file + ": " + err);
-								}
-								callback();
-							});
-						},
-						function(err) {
-							// complete, unlock and remove transaction from memory
-							self.unlock( 'C|'+path );
-							self._transUnlock(path);
-							self.transactions[path].keys = {}; // release memory
-							self.transactions[path].values = {}; // release memory
-							self.transactions[path].queue = []; // release memory
-							delete self.transactions[path];
-							
-							self.logDebug(3, "Transaction rollback complete: " + trans.id, { path: path });
-							callback();
-						}
-					); // forEachOfLimit
+					self.logDebug(3, "Transaction rollback complete: " + trans.id, { path: path });
+					callback();
 				}); // fs.unlink
 			} // done with log
 		); // fileEachLine