pixl-server-storage 3.1.12 → 3.1.14

package/README.md

@@ -426,7 +426,7 @@ If you want to use [Amazon S3](http://aws.amazon.com/s3/) as a backing store, co
 		"region": "us-west-1",
 		"credentials": {
 			"accessKeyId": "YOUR_AMAZON_ACCESS_KEY", 
-			"secretAccessKey": "YOUR_AMAZON_SECRET_KEY", 
+			"secretAccessKey": "YOUR_AMAZON_SECRET_KEY"
 		}
 	},
 	"S3": {

package/docs/API.md

@@ -17,6 +17,7 @@ var storage = server.Storage;
 	* [putStream](#putstream)
 	* [get](#get)
 	* [getMulti](#getmulti)
+	* [getBuffer](#getbuffer)
 	* [getStream](#getstream)
 	* [getStreamRange](#getstreamrange)
 	* [head](#head)
@@ -60,6 +61,8 @@ var storage = server.Storage;
 	* [hashPutMulti](#hashputmulti)
 	* [hashGet](#hashget)
 	* [hashGetMulti](#hashgetmulti)
+	* [hashUpdate](#hashupdate)
+	* [hashUpdateMulti](#hashupdatemulti)
 	* [hashEach](#hasheach)
 	* [hashEachSync](#hasheachsync)
 	* [hashEachPage](#hasheachpage)
@@ -180,13 +183,27 @@ storage.getMulti( ['test1', 'test2', 'test3'], function(err, values) {
 
 Note that if *any* of the records fail, the entire operation fails, and the first error is passed to your callback.
 
+## getBuffer
+
+```javascript
+storage.getBuffer( KEY, CALLBACK );
+```
+
+The `getBuffer()` method retrieves a [Buffer](https://nodejs.org/api/buffer.html) to a given record's data, regardless if the key points to a JSON record or a binary record.  Your callback function is passed an error if one occurred, and the buffer value for the given record.  Example:
+
+```javascript
+storage.getBuffer( 'test1', function(err, buf) {
+	if (err) throw err;
+} );
+```
+
 ## getStream
 
 ```javascript
 storage.getStream( KEY, CALLBACK );
 ```
 
-The `getStream()` method retrieves a [readable stream](https://nodejs.org/api/stream.html#class-streamreadable) to a given record's data, so it can be read or piped to a writable stream.  This is for very large records, so nothing is loaded into memory.  Example of spooling to a local file:
+The `getStream()` method retrieves a [readable stream](https://nodejs.org/api/stream.html#readable-streams) to a given record's data, so it can be read or piped to a writable stream.  This is for very large records, so nothing is loaded into memory.  Example of spooling to a local file:
 
 ```javascript
 var fs = require('fs');
@@ -982,6 +999,44 @@ storage.hashGetMulti( 'users', ['bsanders', 'hclinton', 'dtrump'], function(err,
 } );
 ```
 
+## hashUpdate
+
+```javascript
+storage.hashUpdate( PATH, KEY, UPDATES, CALLBACK );
+```
+
+The `hashUpdate()` method updates an existing key/value pair in a hash.  The `PATH` specifies the main storage path of the hash, and the hash key itself is identified by `KEY`, which should be a string.  The `UPDATES` must be an object, but it can contain sparse keys.  Furthermore, it can contain dot or slash delimited paths, to update inner nested keys.  The updates are essentially applied atop the exiting record, merging and replacing (overwriting) where appropriate.  Example:
+
+```javascript
+storage.hashUpdate( 'users', 'bsanders', { age: 81 }, function(err) {
+	if (err) throw err;
+} );
+```
+
+This would update Bernie's age to 81 without affecting any of the other properties in his user record.
+
+## hashUpdateMulti
+
+```javascript
+storage.hashUpdateMulti( PATH, RECORDS, CALLBACK );
+```
+
+The `hashUpdateMulti()` method updates multiple key/value pairs in a hash.  The `PATH` specifies the main storage path of the hash, and `RECORDS` should be an object containing all the keys and values you want to update.  See [hashUpdate()](#hashupdate) for details on the update format.
+
+```js
+var updates = {
+	"bsanders": { age: 81 },
+	"hclinton": { age: 75 },
+	"dtrump": { age: 77 }
+};
+
+storage.hashUpdateMulti( 'users', records, function(err) {
+	if (err) throw err;
+} );
+```
+
+This would update the `age` properties in each record, without affecting the other data.
+
 ## hashEach
 
 ```javascript

package/docs/Transactions.md

@@ -200,6 +200,8 @@ When transactions are enabled, many compound storage methods (those that execute
 	- [hashCreate()](API.md#hashcreate)
 	- [hashPut()](API.md#hashput)
 	- [hashPutMulti()](API.md#hashputmulti)
+	- [hashUpdate()](API.md#hashupdate)
+	- [hashUpdateMulti()](API.md#hashupdatemulti)
 	- [hashCopy()](API.md#hashcopy)
 	- [hashRename()](API.md#hashrename)
 	- [hashDeleteMulti()](API.md#hashdeletemulti)

package/engines/Couchbase.js

@@ -203,6 +203,37 @@ module.exports = Class.create({
 		} );
 	},
 	
+	getBuffer: function(key, callback) {
+		// fetch Couchbase buffer given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Fetching Couchbase Object: " + key);
+		
+		this.bucket.get( key, function(err, result) {
+			if (!result) {
+				if (err && (err.code != CouchbaseAPI.errors.keyNotFound)) {
+					// some other error
+					err.message = "Failed to fetch key: " + key + ": " + err.message;
+					self.logError('couchbase', err.message);
+					callback( err, null );
+				}
+				else {
+					// record not found
+					// always use "NoSuchKey" in error code
+					var err = new Error("Failed to fetch key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+					callback( err, null );
+				}
+			}
+			else {
+				var body = result.value;
+				self.logDebug(9, "Binary fetch complete: " + key, '' + body.length + ' bytes');
+				callback( null, body );
+			}
+		} );
+	},
+	
 	getStream: function(key, callback) {
 		// get readable stream to record value given key
 		var self = this;

package/engines/Filesystem.js

@@ -477,6 +477,33 @@ module.exports = Class.create({
 		} );
 	},
 	
+	getBuffer: function(key, callback) {
+		// fetch buffer given key
+		var self = this;
+		var file = this.getFilePath(key);
+		
+		this.logDebug(9, "Fetching Object: " + key, file);
+		
+		fs.readFile(file, function (err, data) {
+			if (err) {
+				if (err.message.match(/ENOENT/)) {
+					err.message = "File not found";
+					err.code = "NoSuchKey";
+				}
+				else {
+					// log fs errors that aren't simple missing files (i.e. I/O errors)
+					self.logError('file', "Failed to read file: " + key + ": " + file + ": " + err.message);
+				}
+				
+				err.message = "Failed to fetch key: " + key + ": " + err.message;
+				return callback( err, null );
+			}
+			
+			self.logDebug(9, "Binary fetch complete: " + key, '' + data.length + ' bytes');
+			callback( null, data );
+		} );
+	},
+	
 	getStream: function(key, callback) {
 		// get readable stream to record value given key
 		var self = this;

package/engines/Hybrid.js

@@ -73,6 +73,16 @@ module.exports = Class.create({
 		}
 	},
 	
+	getBuffer: function(key, callback) {
+		// fetch hybrid buffer given key
+		if (this.storage.isBinaryKey(key)) {
+			this.binaryEngine.getBuffer( key, callback );
+		}
+		else {
+			this.docEngine.getBuffer( key, callback );
+		}
+	},
+	
 	getStream: function(key, callback) {
 		// get readable stream to record value given key
 		// streams are binary only!

package/engines/Redis.js

@@ -203,6 +203,36 @@ module.exports = Class.create({
 		} );
 	},
 	
+	getBuffer: function(key, callback) {
+		// get buffer to record value given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Fetching Redis Object: " + key);
+		
+		this.redis.get( key, function(err, result) {
+			if (!result) {
+				if (err) {
+					// an actual error
+					err.message = "Failed to fetch key: " + key + ": " + err;
+					self.logError('redis', ''+err);
+					callback( err, null );
+				}
+				else {
+					// record not found
+					// always use "NoSuchKey" in error code
+					var err = new Error("Failed to fetch key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+					callback( err, null );
+				}
+			}
+			else {
+				self.logDebug(9, "Binary fetch complete: " + key, '' + result.length + ' bytes');
+				callback( null, result );
+			}
+		} );
+	},
+	
 	getStream: function(key, callback) {
 		// get readable stream to record value given key
 		var self = this;

package/engines/RedisCluster.js

@@ -220,6 +220,36 @@ module.exports = Class.create({
 		} );
 	},
 	
+	getBuffer: function(key, callback) {
+		// fetch Redis buffer given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Fetching Redis Object: " + key);
+		
+		this.redis.getBuffer( key, function(err, result) {
+			if (!result) {
+				if (err) {
+					// an actual error
+					err.message = "Failed to fetch key: " + key + ": " + err;
+					self.logError('redis', ''+err);
+					callback( err, null );
+				}
+				else {
+					// record not found
+					// always use "NoSuchKey" in error code
+					var err = new Error("Failed to fetch key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+					callback( err, null );
+				}
+			}
+			else {
+				self.logDebug(9, "Binary fetch complete: " + key, '' + result.length + ' bytes');
+				callback( null, result );
+			}
+		} );
+	},
+	
 	getStream: function(key, callback) {
 		// get readable stream to record value given key
 		var self = this;

package/engines/S3.js

@@ -355,6 +355,49 @@ module.exports = Class.create({
 			} );
 	},
 	
+	getBuffer: function(key, callback) {
+		// fetch s3 buffer given key
+		var self = this;
+		var orig_key = key;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Fetching S3 Object: " + key);
+		
+		var params = Tools.copyHash( this.s3Params );
+		params.Key = this.extKey(key, orig_key);
+		
+		this.s3.send( new S3.GetObjectCommand(params) )
+			.then( function(data) {
+				// stream to buffer
+				streamToBuffer( data.Body, function (err, body) {
+					if (err) {
+						self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
+						return callback(err);
+					}
+					
+					self.logDebug(9, "Binary fetch complete: " + key, '' + body.length + ' bytes');
+					
+					callback( null, body, {
+						mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
+						len: data.ContentLength
+					} );
+				} ); // streamToBuffer
+			} )
+			.catch( function(err) {
+				if ((err.name == 'NoSuchKey') || (err.name == 'NotFound') || (err.code == 'NoSuchKey') || (err.code == 'NotFound')) {
+					// key not found, special case, don't log an error
+					// always include "Not found" in error message
+					err = new Error("Failed to fetch key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+				}
+				else {
+					// some other error
+					self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
+				}
+				process.nextTick( function() { callback( err ); } );
+			} );
+	},
+	
 	getStream: function(key, callback) {
 		// get readable stream to record value given key
 		var self = this;

package/engines/SQLite.js

@@ -227,6 +227,35 @@ module.exports = Class.create({
 		}); // get
 	},
 	
+	getBuffer: function(key, callback) {
+		// fetch SQLite buffer given key
+		var self = this;
+		key = this.prepKey(key);
+		
+		this.logDebug(9, "Fetching SQLite Object: " + key);
+		
+		this.commands.get.get({ $key: key }, function(err, row) {
+			if (err) {
+				// an actual error
+				err.message = "Failed to fetch key: " + key + ": " + err;
+				self.logError('sqlite', '' + err);
+				callback(err);
+			}
+			else if (!row) {
+				// record not found
+				// always use "NoSuchKey" in error code
+				var err = new Error("Failed to fetch key: " + key + ": Not found");
+				err.code = "NoSuchKey";
+				callback( err, null );
+			}
+			else {
+				// success
+				self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
+				callback( null, row.value );
+			}
+		}); // get
+	},
+	
 	getStream: function(key, callback) {
 		// get readable stream to record value given key
 		var self = this;

package/hash.js

@@ -311,7 +311,7 @@ module.exports = Class.create({
 				
 				if (!(state.hkey in data.items)) {
 					// key not found
-					var err = new Error("Failed to fetch key: " + state.hkey + ": Not found");
+					var err = new Error("Failed to fetch hash key: " + state.path + ": " + state.hkey + ": Not found");
 					err.code = "NoSuchKey";
 					return callback(err);
 				}
@@ -325,7 +325,7 @@ module.exports = Class.create({
 		// fetch multiple hash records at once, given array of keys
 		// callback is provided an array of values in matching order to keys
 		var self = this;
-		var records = {};
+		var records = Object.create(null);
 		
 		async.eachLimit(hkeys, this.concurrency, 
 			function(hkey, callback) {
@@ -350,6 +350,101 @@ module.exports = Class.create({
 		);
 	},
 	
+	hashUpdate: function(path, hkey, updates, callback) {
+		// update existing key/value pair in hash table
+		var self = this;
+		if (!path) return callback(new Error("Hash path must be a valid string."));
+		if (!hkey) return callback(new Error("Hash key must be a valid string."));
+		if (!Tools.isaHash(updates)) return callback(new Error("Hash updates must be an object."));
+		
+		this.logDebug(9, "Updating hash key: " + path + ": " + hkey, this.debugLevel(10) ? updates : null);
+		
+		// lock hash for this
+		this._hashLock(path, true, function() {
+			
+			// load header, do not create new
+			self._hashLoad(path, false, function(err, hash) {
+				if (err) {
+					self._hashUnlock(path);
+					return callback(err);
+				}
+				
+				var state = {
+					path: path,
+					data_path: path + '/data',
+					hkey: ''+hkey,
+					updates: updates,
+					hash: hash,
+					index_depth: -1,
+					key_digest: Tools.digestHex(hkey, 'md5')
+				};
+				
+				self._hashUpdateKey(state, function(err) {
+					// done
+					self._hashUnlock(path);
+					return callback(err);
+				}); // _hashPutKey
+			}); // load
+		}); // lock
+	},
+	
+	_hashUpdateKey: function(state, callback) {
+		// internal hash update method, store at one hashing level
+		// recurse for deeper indexes
+		var self = this;
+		
+		self.get(state.data_path, function(err, data) {
+			if (err) data = { type: 'hash_page', length: 0, items: {} };
+			
+			if (data.type == 'hash_index') {
+				// recurse for deeper level
+				state.index_depth++;
+				state.data_path += '/' + state.key_digest.substring(state.index_depth, state.index_depth + 1);
+				return self._hashUpdateKey(state, callback);
+			}
+			else {
+				// got page, our key should be at this level
+				data.items = Tools.copyHashRemoveProto( data.items );
+				
+				if (!(state.hkey in data.items)) {
+					// key not found
+					var err = new Error("Failed to fetch hash key: " + state.path + ": " + state.hkey + ": Not found");
+					err.code = "NoSuchKey";
+					return callback(err);
+				}
+				
+				var hvalue = data.items[state.hkey];
+				
+				// apply updates directly to forehead
+				for (var key in state.updates) {
+					Tools.setPath( hvalue, key, state.updates[key] );
+				}
+				
+				// save page
+				self.put(state.data_path, data, callback);
+			} // hash_page
+		}); // get
+	},
+	
+	hashUpdateMulti: function(path, records, callback) {
+		// update multiple hash records at once, given object of keys and values
+		// need concurrency limit of 1 because hashUpdate locks
+		var self = this;
+		
+		async.eachLimit(Object.keys(records), 1, 
+			function(hkey, callback) {
+				// iterator for each key
+				self.hashUpdate(path, hkey, records[hkey], function(err) {
+					callback(err);
+				} );
+			}, 
+			function(err) {
+				// all keys updated
+				callback(err);
+			}
+		);
+	},
+	
 	hashEachPage: function(path, iterator, callback) {
 		// call user iterator for each populated hash page, data only
 		// iterator will be passed page items hash object

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.1.12",
+	"version": "3.1.14",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",

package/storage.js

@@ -376,6 +376,30 @@ module.exports = Class.create({
 		} );
 	},
 	
+	getBuffer: function(key, callback) {
+		// fetch buffer given key
+		var self = this;
+		if (!this.started) return callback( new Error("Storage has not completed startup.") );
+		
+		key = this.normalizeKey( key );
+		
+		// invoke engine and track perf
+		var pf = this.perf.begin('get');
+		
+		this.engine.getBuffer( key, function(err, value, info) {
+			// get complete
+			var elapsed = pf.end();
+			if (err) return callback(err);
+			
+			self.logTransaction('get', key, {
+				elapsed_ms: elapsed
+			});
+			
+			callback(null, value, info);
+			if (!err) self.emit('get', key, value, info);
+		} );
+	},
+	
 	getStream: function(key, callback) {
 		// fetch value via stream pipe
 		var self = this;

package/test/test-hash.js

@@ -38,7 +38,18 @@ module.exports = {
 			var self = this;
 			test.expect(2);
 			
-			this.storage.hashPut( 'hash1', 'key1', { foo: 'bar', number: 123 }, function(err) {
+			this.storage.hashPut( 'hash1', 'key1', { foo: 'bar', number: 122 }, function(err) {
+				test.ok( !err, "No error storing into hash: " + err );
+				test.ok( Object.keys(self.storage.locks).length == 0, "No more locks leftover in storage" );
+				test.done();
+			} );
+		},
+		
+		function hashUpdate1(test) {
+			var self = this;
+			test.expect(2);
+			
+			this.storage.hashUpdate( 'hash1', 'key1', { number: 123 }, function(err) {
 				test.ok( !err, "No error storing into hash: " + err );
 				test.ok( Object.keys(self.storage.locks).length == 0, "No more locks leftover in storage" );
 				test.done();
@@ -52,8 +63,8 @@ module.exports = {
 			this.storage.hashGet( 'hash1', 'key1', function(err, item) {
 				test.ok( !err, "No error fetching hash key: " + err );
 				test.ok( !!item, "Item is real" );
-				test.ok( item.number == 123, "List item number matches" );
-				test.ok( item.foo == 'bar', "List item value matches" );
+				test.ok( item.number == 123, "Hash item number matches" );
+				test.ok( item.foo == 'bar', "Hash item value matches" );
 				
 				// check internals
 				self.storage.get( 'hash1', function(err, hash) {

package/test/test-main.js

@@ -374,6 +374,33 @@ module.exports = {
 			} );
 		},
 		
+		function testBuffer(test) {
+			test.expect(8);
+			var self = this;
+			var key = 'buftest';
+			var value = { buf: "test" };
+			
+			this.storage.put( key, value, function(err) {
+				test.ok( !err, "No error creating buftest: " + err );
+				
+				self.storage.getBuffer( key, function(err, data) {
+					test.ok( !err, "No error fetching binary: " + err );
+					test.ok( !!data, "Data is true" );
+					test.ok( typeof(data) == 'object', "Data is an object (not a string)" );
+					test.ok( data.length > 0, "Data length is non-zero" );
+					
+					var json = JSON.parse( data.toString() );
+					test.ok( !!json, "Parsed JSON object from buftest" );
+					test.ok( json.buf === "test", "Correct data inside JSON buftest" );
+					
+					self.storage.delete( key, function(err) {
+						test.ok( !err, "No error deleting buftest key: " + err );
+						test.done();
+					} );
+				} );
+			} );
+		},
+		
 		function testStream(test) {
 			test.expect(14);
 			var self = this;

package/transaction.js

@@ -454,7 +454,9 @@ module.exports = Class.create({
 			'listEachPageUpdate',
 			'hashCreate', 
 			'hashPut', 
-			'hashPutMulti', 
+			'hashPutMulti',
+			'hashUpdate',
+			'hashUpdateMulti', 
 			'hashCopy', 
 			'hashRename', 
 			'hashDeleteMulti',