pixl-server-storage 3.0.8 → 3.0.12

package/README.md

@@ -846,7 +846,7 @@ storage.getStream( 'test1.gif', function(err, readStream) {
 } );
 ```
 
-Please note that not all the storage engines support streams natively, so the content may actually be loaded into RAM in the background.  Namely, as of this writing, the Couchbase and Redis APIs does not support streams, so they are currently simulated in those engines.  Streams *are* supported natively in both the Filesystem and Amazon S3 engines.
+Please note that not all the storage engines support streams natively, so the content may actually be loaded into RAM in the background.  Namely, as of this writing, the Couchbase and Redis APIs do not support streams, so they are currently simulated in those engines.  Streams *are* supported natively in both the Filesystem and Amazon S3 engines.
 
 # Expiring Data
 

package/docs/API.md

@@ -18,6 +18,7 @@ var storage = server.Storage;
 	* [get](#get)
 	* [getMulti](#getmulti)
 	* [getStream](#getstream)
+	* [getStreamRange](#getstreamrange)
 	* [head](#head)
 	* [headMulti](#headmulti)
 	* [delete](#delete)
@@ -126,6 +127,8 @@ storage.putMulti( records, function(err) {
 } );
 ```
 
+Note that if any of the individual put operations fail, the entire `putMulti()` function is aborted, and the first error is passed to your callback.  At this point the operation may have been partially successful, with some records written, and others not.  Due to this uncertainty, you may want to use this method inside of a [transaction](./Transactions.md), which can be safely rolled back upon error.
+
 ## putStream
 
 ```javascript
@@ -142,6 +145,8 @@ storage.putStream( 'test1.gif', stream, function(err) {
 } );
 ```
 
+Please note that as of this writing, the `Couchbase`, `Redis` and `RedisCluster` engines have no native stream API, so the `putStream()` method has to load the entire record into memory.
+
 ## get
 
 ```javascript
@@ -173,19 +178,46 @@ storage.getMulti( ['test1', 'test2', 'test3'], function(err, values) {
 } );
 ```
 
+Note that if *any* of the records fail, the entire operation fails, and the first error is passed to your callback.
+
 ## getStream
 
 ```javascript
 storage.getStream( KEY, CALLBACK );
 ```
 
-The `getStream()` method retrieves a [readable stream](https://nodejs.org/api/stream.html#stream_class_stream_readable) to a given record's data, so it can be read or piped to a writable stream.  This is for very large records, so nothing is loaded into memory.  Example of spooling to a local file:
+The `getStream()` method retrieves a [readable stream](https://nodejs.org/api/stream.html#class-streamreadable) to a given record's data, so it can be read or piped to a writable stream.  This is for very large records, so nothing is loaded into memory.  Example of spooling to a local file:
+
+```javascript
+var fs = require('fs');
+var writeStream = fs.createWriteStream('/var/tmp/downloaded.gif');
+
+storage.getStream( 'test1.gif', function(err, readStream, info) {
+	if (err) throw err;
+	writeStream.on('finish', function() {
+		// data is completely written
+	} );
+	readStream.pipe( writeStream );
+} );
+```
+
+As you can see above, your callback is also passed a 3rd argument, which is an object containing the record's full byte length and modification date.  These properties match those returned when you call [head()](#head) (i.e. `len` and `mod`).
+
+Please note that as of this writing, the `Couchbase`, `Redis` and `RedisCluster` engines have no native stream API, so the `getStream()` method has to load the entire record into memory.
+
+## getStreamRange
+
+```javascript
+storage.getStreamRange( KEY, START, END, CALLBACK );
+```
+
+The `getStreamRange()` method retrieves a [readable stream](https://nodejs.org/api/stream.html#class-streamreadable) to a specific slice of a record's data, so it can be read or piped to a writable stream.  The `start` and `end` arguments should be set to the starting and ending byte offset of the slice you want.  Both values are *inclusive*.  Example of spooling bytes `0-99` of a record to a local file:
 
 ```javascript
 var fs = require('fs');
 var writeStream = fs.createWriteStream('/var/tmp/downloaded.gif');
 
-storage.getStream( 'test1.gif', function(err, readStream) {
+storage.getStreamRange( 'test1.gif', 0, 99, function(err, readStream, info) {
 	if (err) throw err;
 	writeStream.on('finish', function() {
 		// data is completely written
@@ -194,6 +226,15 @@ storage.getStream( 'test1.gif', function(err, readStream) {
 } );
 ```
 
+As you can see in the example above, your callback is also passed a 3rd argument, which is an object containing the record's full byte length and modification date.  These properties match those returned when you call [head()](#head) (i.e. `len` and `mod`).  The `len` is not affected by the `start` and `end` range -- it is always the full byte length of the record.
+
+The API supports two special range cases:
+
+- If `start` is valid but `end` is `NaN`, it is assumed you want the entire record starting at `start` offset.
+- If `start` is `NaN` but `end` is valid, it is assumed you want `end` bytes from the end of the record.
+
+Please note that as of this writing, the `Couchbase`, `Redis` and `RedisCluster` engines have no native stream API, so the `getStreamRange()` method has to load the entire record into memory.
+
 ## head
 
 ```javascript
@@ -217,7 +258,7 @@ storage.head( 'test1', function(err, data) {
 } );
 ```
 
-Please note that as of this writing, the `Couchbase` engine has no native API, so the `head()` method has to load the entire record.  It does return the record size in to the `len` property, but there is no way to retrieve the last modified date.
+Please note that as of this writing, the `Couchbase`, `Redis` and `RedisCluster` engines have no native head API, so the `head()` method has to load the entire record.  It does return the record size in to the `len` property, but there is no way to retrieve the last modified date.
 
 ## headMulti
 

package/engines/Couchbase.js

@@ -224,7 +224,7 @@ module.exports = Class.create({
 			}
 			
 			var stream = new BufferStream(buf);
-			callback(null, stream);
+			callback(null, stream, { mod: 1, len: buf.length });
 		} );
 	},
 	
@@ -248,9 +248,23 @@ module.exports = Class.create({
 				return callback( err, null );
 			}
 			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if ((start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
+				return;
+			}
+			
 			var range = buf.slice(start, end + 1);
 			var stream = new BufferStream(range);
-			callback(null, stream);
+			callback(null, stream, { mod: 1, len: buf.length });
 		} );
 	},
 	

package/engines/Filesystem.js

@@ -5,7 +5,6 @@
 var path = require('path');
 var fs = require('fs');
 var async = require('async');
-var mkdirp = require('mkdirp');
 var crypto = require('crypto');
 
 var Class = require("pixl-class");
@@ -13,6 +12,8 @@ var Component = require("pixl-server/component");
 var Tools = require("pixl-tools");
 var Cache = require("pixl-cache");
 
+var mkdirp = Tools.mkdirp;
+
 module.exports = Class.create({
 	
 	__name: 'Filesystem',
@@ -449,7 +450,10 @@ module.exports = Class.create({
 			// create read stream
 			var inp = fs.createReadStream( file );
 			
-			callback( null, inp );
+			callback( null, inp, {
+				mod: Math.floor(stats.mtime.getTime() / 1000),
+				len: stats.size
+			} );
 		} );
 	},
 	
@@ -476,10 +480,27 @@ module.exports = Class.create({
 				return callback( err, null );
 			}
 			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = stats.size - end;
+				end = stats.size ? stats.size - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = stats.size ? stats.size - 1 : 0;
+			}
+			if ((start < 0) || (start >= stats.size) || (end < start) || (end >= stats.size)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + stats.size + ")"), null );
+				return;
+			}
+			
 			// create read stream
 			var inp = fs.createReadStream( file, { start, end } );
 			
-			callback( null, inp );
+			callback( null, inp, {
+				mod: Math.floor(stats.mtime.getTime() / 1000),
+				len: stats.size
+			} );
 		} );
 	},
 	

package/engines/Redis.js

@@ -227,7 +227,7 @@ module.exports = Class.create({
 			}
 			
 			var stream = new BufferStream(buf);
-			callback(null, stream);
+			callback(null, stream, { mod: 1, len: buf.length });
 		} );
 	},
 	
@@ -252,9 +252,23 @@ module.exports = Class.create({
 				return callback( err, null );
 			}
 			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if ((start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
+				return;
+			}
+			
 			var range = buf.slice(start, end + 1);
 			var stream = new BufferStream(range);
-			callback(null, stream);
+			callback(null, stream, { mod: 1, len: buf.length });
 		} );
 	},
 	

package/engines/RedisCluster.js

@@ -244,7 +244,7 @@ module.exports = Class.create({
 			}
 			
 			var stream = new BufferStream(buf);
-			callback(null, stream);
+			callback(null, stream, { mod: 1, len: buf.length });
 		} );
 	},
 	
@@ -269,9 +269,23 @@ module.exports = Class.create({
 				return callback( err, null );
 			}
 			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if ((start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
+				return;
+			}
+			
 			var range = buf.slice(start, end + 1);
 			var stream = new BufferStream(range);
-			callback(null, stream);
+			callback(null, stream, { mod: 1, len: buf.length });
 		} );
 	},
 	

package/engines/S3.js

@@ -294,33 +294,38 @@ module.exports = Class.create({
 		this.logDebug(9, "Fetching S3 Stream: " + key);
 		
 		var params = { Key: this.extKey(key, orig_key) };
+		var download = self.s3.getObject(params).createReadStream();
+		var proceed = false;
+		
+		download.on('error', function(err) {
+			if (proceed) self.logError('s3', "Failed to download key: " + key + ": " + err.message);
+		});
+		download.once('end', function() {
+			self.logDebug(9, "S3 stream download complete: " + key);
+		} );
+		download.once('close', function() {
+			self.logDebug(9, "S3 stream download closed: " + key);
+		} );
 		
 		// double-callback protection (bug in aws-sdk)
 		var done = false;
-		this.s3.headObject( params, function(err) {
+		this.s3.headObject( params, function(err, data) {
 			if (done) return; else done = true;
 			
 			if (err) {
 				if (err.code != 'NoSuchKey') {
 					self.logError('s3', "Failed to head key: " + key + ": " + err.message);
 				}
+				download.destroy();
 				callback( err, null );
 				return;
 			}
 			
-			var download = self.s3.getObject(params).createReadStream();
-			
-			download.once('error', function(err) {
-				self.logError('s3', "Failed to download key: " + key + ": " + err.message);
-			});
-			download.once('end', function() {
-				self.logDebug(9, "S3 stream download complete: " + key);
-			} );
-			download.once('close', function() {
-				self.logDebug(9, "S3 stream download closed: " + key);
+			proceed = true;
+			callback( null, download, {
+				mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
+				len: data.ContentLength
 			} );
-			
-			callback( null, download );
 		}); // headObject
 	},
 	
@@ -333,34 +338,52 @@ module.exports = Class.create({
 		this.logDebug(9, "Fetching ranged S3 stream: " + key, { start, end });
 		
 		var params = { Key: this.extKey(key, orig_key) };
+		var download = self.s3.getObject( Tools.mergeHashes(params, { Range: "bytes=" + start + "-" + end }) ).createReadStream();
+		var proceed = false;
+		
+		download.on('error', function(err) {
+			if (proceed) self.logError('s3', "Failed to download key: " + key + ": " + err.message);
+		});
+		download.once('end', function() {
+			self.logDebug(9, "S3 stream download complete: " + key);
+		} );
+		download.once('close', function() {
+			self.logDebug(9, "S3 stream download closed: " + key);
+		} );
 		
 		// double-callback protection (bug in aws-sdk)
 		var done = false;
-		this.s3.headObject( params, function(err) {
+		this.s3.headObject( params, function(err, data) {
 			if (done) return; else done = true;
 			
 			if (err) {
 				if (err.code != 'NoSuchKey') {
 					self.logError('s3', "Failed to head key: " + key + ": " + err.message);
 				}
+				download.destroy();
 				callback( err, null );
 				return;
 			}
 			
-			params.Range = "bytes=" + start + "-" + end;
-			var download = self.s3.getObject(params).createReadStream();
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = data.ContentLength - end;
+				end = data.ContentLength ? data.ContentLength - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = data.ContentLength ? data.ContentLength - 1 : 0;
+			}
+			if ((start < 0) || (start >= data.ContentLength) || (end < start) || (end >= data.ContentLength)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + data.ContentLength + ")"), null );
+				return;
+			}
 			
-			download.once('error', function(err) {
-				self.logError('s3', "Failed to download key: " + key + ": " + err.message);
-			});
-			download.once('end', function() {
-				self.logDebug(9, "S3 stream download complete: " + key);
-			} );
-			download.once('close', function() {
-				self.logDebug(9, "S3 stream download closed: " + key);
+			proceed = true;
+			callback( null, download, {
+				mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
+				len: data.ContentLength
 			} );
-			
-			callback( null, download );
 		}); // headObject
 	},
 	

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.0.8",
+	"version": "3.0.12",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -27,8 +27,7 @@
 		"pixl-perf": "^1.0.0",
 		"pixl-cache": "^1.0.1",
 		"async": "2.6.0",
-		"mkdirp": "0.5.1",
-		"nearley": "2.11.0",
+		"nearley": "2.20.1",
 		"moo": "0.4.3",
 		"porter-stemmer": "0.9.1",
 		"unidecode": "0.1.8",

package/test/test-main.js

@@ -375,7 +375,7 @@ module.exports = {
 		},
 		
 		function testStream(test) {
-			test.expect(11);
+			test.expect(14);
 			var self = this;
 			
 			var key = 'spacer-stream.gif';
@@ -395,10 +395,13 @@ module.exports = {
 				var tempFile = __dirname + '/' + filename + '.streamtemp';
 				var outStream = fs.createWriteStream( tempFile );
 				
-				self.storage.getStream( key, function(err, storageStream) {
+				self.storage.getStream( key, function(err, storageStream, streamInfo) {
 					test.ok( !err, "No error fetching stream: " + err );
 					test.ok( !!storageStream, "Got storage stream as 2nd arg");
 					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
+					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
+					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
 					
 					outStream.on('finish', function() {
 						var newSpacerBuf = fs.readFileSync( tempFile );
@@ -420,6 +423,147 @@ module.exports = {
 			} ); // putStream
 		},
 		
+		function testStreamRange(test) {
+			// grab a range from within a stream, with both start and end specified
+			test.expect(14);
+			var self = this;
+			
+			var key = 'spacer-stream.gif';
+			var filename = 'spacer.gif';
+			var spacerBuf = fs.readFileSync( __dirname + '/' + filename );
+			var spacerStream = fs.createReadStream( __dirname + '/' + filename );
+			
+			test.ok( !!spacerBuf, "Got buffer from file" );
+			test.ok( typeof(spacerBuf) == 'object', "Buffer is an object" );
+			test.ok( spacerBuf.length > 0, "Buffer has size" );
+			test.ok( !!spacerStream, "Got read stream" );
+			
+			this.storage.putStream( key, spacerStream, function(err) {
+				test.ok( !err, "No error creating stream: " + err );
+				
+				var tempFile = __dirname + '/' + filename + '.streamtemp';
+				var outStream = fs.createWriteStream( tempFile );
+				
+				self.storage.getStreamRange( key, 0, 5, function(err, storageStream, streamInfo) {
+					test.ok( !err, "No error fetching stream: " + err );
+					test.ok( !!storageStream, "Got storage stream as 2nd arg");
+					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
+					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
+					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
+					
+					outStream.on('finish', function() {
+						var newSpacerBuf = fs.readFileSync( tempFile );
+						test.ok( newSpacerBuf.length == 6, "Stream length is correct" );
+						test.ok( newSpacerBuf.toString() == "GIF89a", "Range buffer content is correct" );
+						
+						self.storage.delete( key, function(err) {
+							test.ok( !err, "No error deleting stream key: " + err );
+							fs.unlinkSync( tempFile );
+							test.done();
+						} ); // delete
+					} ); // stream finish
+					
+					storageStream.pipe( outStream );
+					
+				} ); // getStream
+			} ); // putStream
+		},
+		
+		function testStreamRangeStart(test) {
+			// grab a range from within a stream, with end missing
+			test.expect(14);
+			var self = this;
+			
+			var key = 'spacer-stream.gif';
+			var filename = 'spacer.gif';
+			var spacerBuf = fs.readFileSync( __dirname + '/' + filename );
+			var spacerStream = fs.createReadStream( __dirname + '/' + filename );
+			
+			test.ok( !!spacerBuf, "Got buffer from file" );
+			test.ok( typeof(spacerBuf) == 'object', "Buffer is an object" );
+			test.ok( spacerBuf.length > 0, "Buffer has size" );
+			test.ok( !!spacerStream, "Got read stream" );
+			
+			this.storage.putStream( key, spacerStream, function(err) {
+				test.ok( !err, "No error creating stream: " + err );
+				
+				var tempFile = __dirname + '/' + filename + '.streamtemp';
+				var outStream = fs.createWriteStream( tempFile );
+				
+				self.storage.getStreamRange( key, 20, NaN, function(err, storageStream, streamInfo) {
+					test.ok( !err, "No error fetching stream: " + err );
+					test.ok( !!storageStream, "Got storage stream as 2nd arg");
+					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
+					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
+					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
+					
+					outStream.on('finish', function() {
+						var newSpacerBuf = fs.readFileSync( tempFile );
+						test.ok( newSpacerBuf.length == 23, "Stream range length is correct" );
+						test.ok( newSpacerBuf.equals( spacerBuf.slice(20) ), "Range buffer content is correct" );
+						
+						self.storage.delete( key, function(err) {
+							test.ok( !err, "No error deleting stream key: " + err );
+							fs.unlinkSync( tempFile );
+							test.done();
+						} ); // delete
+					} ); // stream finish
+					
+					storageStream.pipe( outStream );
+					
+				} ); // getStream
+			} ); // putStream
+		},
+		
+		function testStreamRangeEnd(test) {
+			// grab a range from within a stream, with start missing
+			test.expect(14);
+			var self = this;
+			
+			var key = 'spacer-stream.gif';
+			var filename = 'spacer.gif';
+			var spacerBuf = fs.readFileSync( __dirname + '/' + filename );
+			var spacerStream = fs.createReadStream( __dirname + '/' + filename );
+			
+			test.ok( !!spacerBuf, "Got buffer from file" );
+			test.ok( typeof(spacerBuf) == 'object', "Buffer is an object" );
+			test.ok( spacerBuf.length > 0, "Buffer has size" );
+			test.ok( !!spacerStream, "Got read stream" );
+			
+			this.storage.putStream( key, spacerStream, function(err) {
+				test.ok( !err, "No error creating stream: " + err );
+				
+				var tempFile = __dirname + '/' + filename + '.streamtemp';
+				var outStream = fs.createWriteStream( tempFile );
+				
+				self.storage.getStreamRange( key, NaN, 10, function(err, storageStream, streamInfo) {
+					test.ok( !err, "No error fetching stream: " + err );
+					test.ok( !!storageStream, "Got storage stream as 2nd arg");
+					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
+					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
+					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
+					
+					outStream.on('finish', function() {
+						var newSpacerBuf = fs.readFileSync( tempFile );
+						test.ok( newSpacerBuf.length == 10, "Stream range length is correct" );
+						test.ok( newSpacerBuf.equals( spacerBuf.slice(43 - 10) ), "Range buffer content is correct" );
+						
+						self.storage.delete( key, function(err) {
+							test.ok( !err, "No error deleting stream key: " + err );
+							fs.unlinkSync( tempFile );
+							test.done();
+						} ); // delete
+					} ); // stream finish
+					
+					storageStream.pipe( outStream );
+					
+				} ); // getStream
+			} ); // putStream
+		},
+		
 		function testPutMulti(test) {
 			// test storing multiple keys at once
 			test.expect(1);

package/transaction.js

@@ -8,9 +8,9 @@ var Path = require("path");
 var cp = require("child_process");
 var os = require("os");
 var async = require('async');
-var mkdirp = require('mkdirp');
 var Class = require("pixl-class");
 var Tools = require("pixl-tools");
+var mkdirp = Tools.mkdirp;
 
 // Transaction support is implemented as a mixin to Storage
 // Config Keys: