pixl-server-storage 3.0.9 → 3.0.13

package/docs/API.md

@@ -226,7 +226,12 @@ storage.getStreamRange( 'test1.gif', 0, 99, function(err, readStream, info) {
 } );
 ```
 
-As you can see above, your callback is also passed a 3rd argument, which is an object containing the record's full byte length and modification date.  These properties match those returned when you call [head()](#head) (i.e. `len` and `mod`).  The `len` is not affected by the `start` and `end` range -- it is always the full byte length of the record.
+As you can see in the example above, your callback is also passed a 3rd argument, which is an object containing the record's full byte length and modification date.  These properties match those returned when you call [head()](#head) (i.e. `len` and `mod`).  The `len` is not affected by the `start` and `end` range -- it is always the full byte length of the record.
+
+The API supports two special range cases:
+
+- If `start` is valid but `end` is `NaN`, it is assumed you want the entire record starting at `start` offset.
+- If `start` is `NaN` but `end` is valid, it is assumed you want `end` bytes from the end of the record.
 
 Please note that as of this writing, the `Couchbase`, `Redis` and `RedisCluster` engines have no native stream API, so the `getStreamRange()` method has to load the entire record into memory.
 

package/engines/Couchbase.js

@@ -248,6 +248,20 @@ module.exports = Class.create({
 				return callback( err, null );
 			}
 			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
+				return;
+			}
+			
 			var range = buf.slice(start, end + 1);
 			var stream = new BufferStream(range);
 			callback(null, stream, { mod: 1, len: buf.length });

package/engines/Filesystem.js

@@ -5,7 +5,6 @@
 var path = require('path');
 var fs = require('fs');
 var async = require('async');
-var mkdirp = require('mkdirp');
 var crypto = require('crypto');
 
 var Class = require("pixl-class");
@@ -13,6 +12,8 @@ var Component = require("pixl-server/component");
 var Tools = require("pixl-tools");
 var Cache = require("pixl-cache");
 
+var mkdirp = Tools.mkdirp;
+
 module.exports = Class.create({
 	
 	__name: 'Filesystem',
@@ -479,6 +480,20 @@ module.exports = Class.create({
 				return callback( err, null );
 			}
 			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = stats.size - end;
+				end = stats.size ? stats.size - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = stats.size ? stats.size - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= stats.size) || (end < start) || (end >= stats.size)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + stats.size + ")"), null );
+				return;
+			}
+			
 			// create read stream
 			var inp = fs.createReadStream( file, { start, end } );
 			

package/engines/Redis.js

@@ -252,6 +252,20 @@ module.exports = Class.create({
 				return callback( err, null );
 			}
 			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
+				return;
+			}
+			
 			var range = buf.slice(start, end + 1);
 			var stream = new BufferStream(range);
 			callback(null, stream, { mod: 1, len: buf.length });

package/engines/RedisCluster.js

@@ -269,6 +269,20 @@ module.exports = Class.create({
 				return callback( err, null );
 			}
 			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
+				return;
+			}
+			
 			var range = buf.slice(start, end + 1);
 			var stream = new BufferStream(range);
 			callback(null, stream, { mod: 1, len: buf.length });

package/engines/S3.js

@@ -338,7 +338,14 @@ module.exports = Class.create({
 		this.logDebug(9, "Fetching ranged S3 stream: " + key, { start, end });
 		
 		var params = { Key: this.extKey(key, orig_key) };
-		var download = self.s3.getObject( Tools.mergeHashes(params, { Range: "bytes=" + start + "-" + end }) ).createReadStream();
+		
+		// convert start/end to HTTP range header string
+		var range = "bytes=";
+		if (!isNaN(start)) range += start;
+		range += '-';
+		if (!isNaN(end)) range += end;
+		
+		var download = self.s3.getObject( Tools.mergeHashes(params, { Range: range }) ).createReadStream();
 		var proceed = false;
 		
 		download.on('error', function(err) {
@@ -365,6 +372,20 @@ module.exports = Class.create({
 				return;
 			}
 			
+			// validate byte range, now that we have the head info
+			if (isNaN(start) && !isNaN(end)) {
+				start = data.ContentLength - end;
+				end = data.ContentLength ? data.ContentLength - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = data.ContentLength ? data.ContentLength - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= data.ContentLength) || (end < start) || (end >= data.ContentLength)) {
+				download.destroy();
+				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + data.ContentLength + ")"), null );
+				return;
+			}
+			
 			proceed = true;
 			callback( null, download, {
 				mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.0.9",
+	"version": "3.0.13",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -27,8 +27,7 @@
 		"pixl-perf": "^1.0.0",
 		"pixl-cache": "^1.0.1",
 		"async": "2.6.0",
-		"mkdirp": "0.5.1",
-		"nearley": "2.11.0",
+		"nearley": "2.20.1",
 		"moo": "0.4.3",
 		"porter-stemmer": "0.9.1",
 		"unidecode": "0.1.8",

package/test/test-main.js

@@ -424,6 +424,7 @@ module.exports = {
 		},
 		
 		function testStreamRange(test) {
+			// grab a range from within a stream, with both start and end specified
 			test.expect(14);
 			var self = this;
 			
@@ -469,6 +470,100 @@ module.exports = {
 			} ); // putStream
 		},
 		
+		function testStreamRangeStart(test) {
+			// grab a range from within a stream, with end missing
+			test.expect(14);
+			var self = this;
+			
+			var key = 'spacer-stream.gif';
+			var filename = 'spacer.gif';
+			var spacerBuf = fs.readFileSync( __dirname + '/' + filename );
+			var spacerStream = fs.createReadStream( __dirname + '/' + filename );
+			
+			test.ok( !!spacerBuf, "Got buffer from file" );
+			test.ok( typeof(spacerBuf) == 'object', "Buffer is an object" );
+			test.ok( spacerBuf.length > 0, "Buffer has size" );
+			test.ok( !!spacerStream, "Got read stream" );
+			
+			this.storage.putStream( key, spacerStream, function(err) {
+				test.ok( !err, "No error creating stream: " + err );
+				
+				var tempFile = __dirname + '/' + filename + '.streamtemp';
+				var outStream = fs.createWriteStream( tempFile );
+				
+				self.storage.getStreamRange( key, 20, NaN, function(err, storageStream, streamInfo) {
+					test.ok( !err, "No error fetching stream: " + err );
+					test.ok( !!storageStream, "Got storage stream as 2nd arg");
+					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
+					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
+					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
+					
+					outStream.on('finish', function() {
+						var newSpacerBuf = fs.readFileSync( tempFile );
+						test.ok( newSpacerBuf.length == 23, "Stream range length is correct" );
+						test.ok( newSpacerBuf.equals( spacerBuf.slice(20) ), "Range buffer content is correct" );
+						
+						self.storage.delete( key, function(err) {
+							test.ok( !err, "No error deleting stream key: " + err );
+							fs.unlinkSync( tempFile );
+							test.done();
+						} ); // delete
+					} ); // stream finish
+					
+					storageStream.pipe( outStream );
+					
+				} ); // getStream
+			} ); // putStream
+		},
+		
+		function testStreamRangeEnd(test) {
+			// grab a range from within a stream, with start missing
+			test.expect(14);
+			var self = this;
+			
+			var key = 'spacer-stream.gif';
+			var filename = 'spacer.gif';
+			var spacerBuf = fs.readFileSync( __dirname + '/' + filename );
+			var spacerStream = fs.createReadStream( __dirname + '/' + filename );
+			
+			test.ok( !!spacerBuf, "Got buffer from file" );
+			test.ok( typeof(spacerBuf) == 'object', "Buffer is an object" );
+			test.ok( spacerBuf.length > 0, "Buffer has size" );
+			test.ok( !!spacerStream, "Got read stream" );
+			
+			this.storage.putStream( key, spacerStream, function(err) {
+				test.ok( !err, "No error creating stream: " + err );
+				
+				var tempFile = __dirname + '/' + filename + '.streamtemp';
+				var outStream = fs.createWriteStream( tempFile );
+				
+				self.storage.getStreamRange( key, NaN, 10, function(err, storageStream, streamInfo) {
+					test.ok( !err, "No error fetching stream: " + err );
+					test.ok( !!storageStream, "Got storage stream as 2nd arg");
+					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
+					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
+					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
+					
+					outStream.on('finish', function() {
+						var newSpacerBuf = fs.readFileSync( tempFile );
+						test.ok( newSpacerBuf.length == 10, "Stream range length is correct" );
+						test.ok( newSpacerBuf.equals( spacerBuf.slice(43 - 10) ), "Range buffer content is correct" );
+						
+						self.storage.delete( key, function(err) {
+							test.ok( !err, "No error deleting stream key: " + err );
+							fs.unlinkSync( tempFile );
+							test.done();
+						} ); // delete
+					} ); // stream finish
+					
+					storageStream.pipe( outStream );
+					
+				} ); // getStream
+			} ); // putStream
+		},
+		
 		function testPutMulti(test) {
 			// test storing multiple keys at once
 			test.expect(1);

package/transaction.js

@@ -8,9 +8,9 @@ var Path = require("path");
 var cp = require("child_process");
 var os = require("os");
 var async = require('async');
-var mkdirp = require('mkdirp');
 var Class = require("pixl-class");
 var Tools = require("pixl-tools");
+var mkdirp = Tools.mkdirp;
 
 // Transaction support is implemented as a mixin to Storage
 // Config Keys: