pixl-server-storage 3.0.10 → 3.0.14

package/docs/API.md

@@ -226,7 +226,12 @@ storage.getStreamRange( 'test1.gif', 0, 99, function(err, readStream, info) {
 } );
 ```
 
-As you can see above, your callback is also passed a 3rd argument, which is an object containing the record's full byte length and modification date.  These properties match those returned when you call [head()](#head) (i.e. `len` and `mod`).  The `len` is not affected by the `start` and `end` range -- it is always the full byte length of the record.
+As you can see in the example above, your callback is also passed a 3rd argument, which is an object containing the record's full byte length and modification date.  These properties match those returned when you call [head()](#head) (i.e. `len` and `mod`).  The `len` is not affected by the `start` and `end` range -- it is always the full byte length of the record.
+
+The API supports two special range cases:
+
+- If `start` is valid but `end` is `NaN`, it is assumed you want the entire record starting at `start` offset.
+- If `start` is `NaN` but `end` is valid, it is assumed you want `end` bytes from the end of the record.
 
 Please note that as of this writing, the `Couchbase`, `Redis` and `RedisCluster` engines have no native stream API, so the `getStreamRange()` method has to load the entire record into memory.
 

package/engines/Couchbase.js

@@ -249,7 +249,14 @@ module.exports = Class.create({
 			}
 			
 			// validate byte range, now that we have the head info
-			if ((start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
 				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
 				return;

package/engines/Filesystem.js

@@ -5,7 +5,6 @@
 var path = require('path');
 var fs = require('fs');
 var async = require('async');
-var mkdirp = require('mkdirp');
 var crypto = require('crypto');
 
 var Class = require("pixl-class");
@@ -13,6 +12,8 @@ var Component = require("pixl-server/component");
 var Tools = require("pixl-tools");
 var Cache = require("pixl-cache");
 
+var mkdirp = Tools.mkdirp;
+
 module.exports = Class.create({
 	
 	__name: 'Filesystem',
@@ -480,7 +481,14 @@ module.exports = Class.create({
 			}
 			
 			// validate byte range, now that we have the head info
-			if ((start < 0) || (start >= stats.size) || (end < start) || (end >= stats.size)) {
+			if (isNaN(start) && !isNaN(end)) {
+				start = stats.size - end;
+				end = stats.size ? stats.size - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = stats.size ? stats.size - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= stats.size) || (end < start) || (end >= stats.size)) {
 				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + stats.size + ")"), null );
 				return;

package/engines/Redis.js

@@ -39,10 +39,8 @@ module.exports = Class.create({
 		
 		this.keyPrefix = (r_config.keyPrefix || '').replace(/^\//, '');
 		if (this.keyPrefix && !this.keyPrefix.match(/\/$/)) this.keyPrefix += '/';
-		delete r_config.keyPrefix;
 		
 		this.keyTemplate = (r_config.keyTemplate || '').replace(/^\//, '').replace(/\/$/, '');
-		delete r_config.keyTemplate;
 		
 		r_config.return_buffers = true;
 		r_config.retry_strategy = function(opts) {
@@ -50,7 +48,7 @@ module.exports = Class.create({
 			return Math.min(opts.attempt * 100, 3000);
 		};
 		
-		this.redis = Redis.createClient(r_config);
+		this.redis = Redis.createClient( Tools.copyHashRemoveKeys(r_config, { keyPrefix:1, keyTemplate:1 }) );
 		
 		this.redis.on('error', function(err) {
 			if (!self.storage.started) return callback(err);
@@ -253,7 +251,14 @@ module.exports = Class.create({
 			}
 			
 			// validate byte range, now that we have the head info
-			if ((start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
 				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
 				return;

package/engines/RedisCluster.js

@@ -44,10 +44,8 @@ module.exports = Class.create({
 		
 		this.keyPrefix = (r_config.keyPrefix || '').replace(/^\//, '');
 		if (this.keyPrefix && !this.keyPrefix.match(/\/$/)) this.keyPrefix += '/';
-		delete r_config.keyPrefix;
 		
 		this.keyTemplate = (r_config.keyTemplate || '').replace(/^\//, '').replace(/\/$/, '');
-		delete r_config.keyTemplate;
 		
 		r_config.clusterOpts.clusterRetryStrategy = function(attempts) {
 			if (attempts > r_config.connectRetries) return false;
@@ -270,7 +268,14 @@ module.exports = Class.create({
 			}
 			
 			// validate byte range, now that we have the head info
-			if ((start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
+			if (isNaN(start) && !isNaN(end)) {
+				start = buf.length - end;
+				end = buf.length ? buf.length - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = buf.length ? buf.length - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
 				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
 				return;

package/engines/S3.js

@@ -37,13 +37,9 @@ module.exports = Class.create({
 		
 		this.keyPrefix = (s3_config.keyPrefix || '').replace(/^\//, '');
 		if (this.keyPrefix && !this.keyPrefix.match(/\/$/)) this.keyPrefix += '/';
-		delete s3_config.keyPrefix;
 		
 		this.keyTemplate = (s3_config.keyTemplate || '').replace(/^\//, '').replace(/\/$/, '');
-		delete s3_config.keyTemplate;
-		
 		this.fileExtensions = !!s3_config.fileExtensions;
-		delete s3_config.fileExtensions;
 		
 		if (this.debugLevel(10)) {
 			// S3 has a logger API but it's extremely verbose -- restrict to level 10 only
@@ -70,7 +66,7 @@ module.exports = Class.create({
 		delete s3_config.cache;
 		
 		AWS.config.update( aws_config );
-		this.s3 = new AWS.S3( s3_config );
+		this.s3 = new AWS.S3( Tools.copyHashRemoveKeys(s3_config, { keyPrefix:1, keyTemplate:1, fileExtensions:1, cache:1 }) );
 	},
 	
 	prepKey: function(key) {
@@ -338,7 +334,14 @@ module.exports = Class.create({
 		this.logDebug(9, "Fetching ranged S3 stream: " + key, { start, end });
 		
 		var params = { Key: this.extKey(key, orig_key) };
-		var download = self.s3.getObject( Tools.mergeHashes(params, { Range: "bytes=" + start + "-" + end }) ).createReadStream();
+		
+		// convert start/end to HTTP range header string
+		var range = "bytes=";
+		if (!isNaN(start)) range += start;
+		range += '-';
+		if (!isNaN(end)) range += end;
+		
+		var download = self.s3.getObject( Tools.mergeHashes(params, { Range: range }) ).createReadStream();
 		var proceed = false;
 		
 		download.on('error', function(err) {
@@ -366,7 +369,14 @@ module.exports = Class.create({
 			}
 			
 			// validate byte range, now that we have the head info
-			if ((start < 0) || (start >= data.ContentLength) || (end < start) || (end >= data.ContentLength)) {
+			if (isNaN(start) && !isNaN(end)) {
+				start = data.ContentLength - end;
+				end = data.ContentLength ? data.ContentLength - 1 : 0;
+			} 
+			else if (!isNaN(start) && isNaN(end)) {
+				end = data.ContentLength ? data.ContentLength - 1 : 0;
+			}
+			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= data.ContentLength) || (end < start) || (end >= data.ContentLength)) {
 				download.destroy();
 				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + data.ContentLength + ")"), null );
 				return;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.0.10",
+	"version": "3.0.14",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -27,8 +27,7 @@
 		"pixl-perf": "^1.0.0",
 		"pixl-cache": "^1.0.1",
 		"async": "2.6.0",
-		"mkdirp": "0.5.1",
-		"nearley": "2.11.0",
+		"nearley": "2.20.1",
 		"moo": "0.4.3",
 		"porter-stemmer": "0.9.1",
 		"unidecode": "0.1.8",

package/test/test-main.js

@@ -424,6 +424,7 @@ module.exports = {
 		},
 		
 		function testStreamRange(test) {
+			// grab a range from within a stream, with both start and end specified
 			test.expect(14);
 			var self = this;
 			
@@ -469,6 +470,100 @@ module.exports = {
 			} ); // putStream
 		},
 		
+		function testStreamRangeStart(test) {
+			// grab a range from within a stream, with end missing
+			test.expect(14);
+			var self = this;
+			
+			var key = 'spacer-stream.gif';
+			var filename = 'spacer.gif';
+			var spacerBuf = fs.readFileSync( __dirname + '/' + filename );
+			var spacerStream = fs.createReadStream( __dirname + '/' + filename );
+			
+			test.ok( !!spacerBuf, "Got buffer from file" );
+			test.ok( typeof(spacerBuf) == 'object', "Buffer is an object" );
+			test.ok( spacerBuf.length > 0, "Buffer has size" );
+			test.ok( !!spacerStream, "Got read stream" );
+			
+			this.storage.putStream( key, spacerStream, function(err) {
+				test.ok( !err, "No error creating stream: " + err );
+				
+				var tempFile = __dirname + '/' + filename + '.streamtemp';
+				var outStream = fs.createWriteStream( tempFile );
+				
+				self.storage.getStreamRange( key, 20, NaN, function(err, storageStream, streamInfo) {
+					test.ok( !err, "No error fetching stream: " + err );
+					test.ok( !!storageStream, "Got storage stream as 2nd arg");
+					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
+					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
+					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
+					
+					outStream.on('finish', function() {
+						var newSpacerBuf = fs.readFileSync( tempFile );
+						test.ok( newSpacerBuf.length == 23, "Stream range length is correct" );
+						test.ok( newSpacerBuf.equals( spacerBuf.slice(20) ), "Range buffer content is correct" );
+						
+						self.storage.delete( key, function(err) {
+							test.ok( !err, "No error deleting stream key: " + err );
+							fs.unlinkSync( tempFile );
+							test.done();
+						} ); // delete
+					} ); // stream finish
+					
+					storageStream.pipe( outStream );
+					
+				} ); // getStream
+			} ); // putStream
+		},
+		
+		function testStreamRangeEnd(test) {
+			// grab a range from within a stream, with start missing
+			test.expect(14);
+			var self = this;
+			
+			var key = 'spacer-stream.gif';
+			var filename = 'spacer.gif';
+			var spacerBuf = fs.readFileSync( __dirname + '/' + filename );
+			var spacerStream = fs.createReadStream( __dirname + '/' + filename );
+			
+			test.ok( !!spacerBuf, "Got buffer from file" );
+			test.ok( typeof(spacerBuf) == 'object', "Buffer is an object" );
+			test.ok( spacerBuf.length > 0, "Buffer has size" );
+			test.ok( !!spacerStream, "Got read stream" );
+			
+			this.storage.putStream( key, spacerStream, function(err) {
+				test.ok( !err, "No error creating stream: " + err );
+				
+				var tempFile = __dirname + '/' + filename + '.streamtemp';
+				var outStream = fs.createWriteStream( tempFile );
+				
+				self.storage.getStreamRange( key, NaN, 10, function(err, storageStream, streamInfo) {
+					test.ok( !err, "No error fetching stream: " + err );
+					test.ok( !!storageStream, "Got storage stream as 2nd arg");
+					test.ok( !!storageStream.pipe, "Storage stream has a pipe");
+					test.ok( !!streamInfo, "Info was provided as the 3rd arg");
+					test.ok( streamInfo.len == 43, "Info has correct data length");
+					test.ok( streamInfo.mod > 0, "Info has a non-zero mod date");
+					
+					outStream.on('finish', function() {
+						var newSpacerBuf = fs.readFileSync( tempFile );
+						test.ok( newSpacerBuf.length == 10, "Stream range length is correct" );
+						test.ok( newSpacerBuf.equals( spacerBuf.slice(43 - 10) ), "Range buffer content is correct" );
+						
+						self.storage.delete( key, function(err) {
+							test.ok( !err, "No error deleting stream key: " + err );
+							fs.unlinkSync( tempFile );
+							test.done();
+						} ); // delete
+					} ); // stream finish
+					
+					storageStream.pipe( outStream );
+					
+				} ); // getStream
+			} ); // putStream
+		},
+		
 		function testPutMulti(test) {
 			// test storing multiple keys at once
 			test.expect(1);

package/transaction.js

@@ -8,9 +8,9 @@ var Path = require("path");
 var cp = require("child_process");
 var os = require("os");
 var async = require('async');
-var mkdirp = require('mkdirp');
 var Class = require("pixl-class");
 var Tools = require("pixl-tools");
+var mkdirp = Tools.mkdirp;
 
 // Transaction support is implemented as a mixin to Storage
 // Config Keys: