pixl-server-storage 3.0.16 → 3.0.19

package/engines/Filesystem.js

@@ -315,6 +315,58 @@ module.exports = Class.create({
 		inp.pipe( outp );
 	},
 	
+	putStreamCustom: function(key, inp, opts, callback) {
+		// store key+stream of data to disk
+		var self = this;
+		var file = this.getFilePath(key);
+		
+		this.logDebug(9, "Storing Binary Stream Object: " + key, file);
+		
+		var dir = path.dirname( file );
+		
+		var temp_file = this.tempDir + '/' + path.basename(file) + '.tmp.' + this.tempFileCounter;
+		this.tempFileCounter = (this.tempFileCounter + 1) % 10000000;
+		
+		// create the write stream to temp file
+		var outp = fs.createWriteStream( temp_file, opts || {} );
+		
+		outp.on('error', function(err) {
+			// failed to write file
+			var msg = "Failed to write file: " + key + ": " + temp_file + ": " + err.message;
+			self.logError('file', msg);
+			return callback( new Error(msg), null );
+		} );
+		
+		outp.on('finish', function() {
+			// make sure parent dirs exist, async
+			self._makeDirs( dir, 0o0775, function(err) {
+				if (err) {
+					// failed to create directory
+					var msg = "Failed to create directory: " + key + ": " + dir + ": " + err.message;
+					self.logError('file', msg);
+					return callback( new Error(msg), null );
+				}
+				
+				// rename temp file to final
+				self._renameFile( temp_file, file, function (err) {
+					if (err) {
+						// failed to write file
+						var msg = "Failed to rename file: " + key + ": " + temp_file + ": " + err.message;
+						self.logError('file', msg);
+						return callback( new Error(msg), null );
+					}
+					
+					// all done
+					self.logDebug(9, "Store operation complete: " + key);
+					callback(null, null);
+				} ); // rename
+			} ); // mkdirp
+		} ); // pipe finish
+		
+		// pipe inp to outp
+		inp.pipe( outp );
+	},
+	
 	head: function(key, callback) {
 		// head value given key
 		var self = this;

package/engines/S3.js

@@ -166,6 +166,33 @@ module.exports = Class.create({
 		} );
 	},
 	
+	putStreamCustom: function(key, inp, opts, callback) {
+		// store key+stream of data to S3, inc options
+		var self = this;
+		var orig_key = key;
+		key = this.prepKey(key);
+		
+		var params = {};
+		params.Key = this.extKey(key, orig_key);
+		params.Body = inp;
+		if (opts) Tools.mergeHashInto(params, opts);
+		
+		this.logDebug(9, "Storing S3 Binary Stream: " + key);
+		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
+		this.s3.upload(params, function(err, data) {
+			if (done) return; else done = true;
+			
+			if (err) {
+				self.logError('s3', "Failed to store stream: " + key + ": " + (err.message || err), err);
+			}
+			else self.logDebug(9, "Stream store complete: " + key);
+			
+			if (callback) callback(err, data);
+		} );
+	},
+	
 	head: function(key, callback) {
 		// head s3 value given key
 		var self = this;
@@ -193,7 +220,14 @@ module.exports = Class.create({
 			if (done) return; else done = true;
 			
 			if (err) {
-				if ((err.code != 'NoSuchKey') && (err.code != 'NotFound')) {
+				if ((err.code == 'NoSuchKey') || (err.code == 'NotFound')) {
+					// key not found, special case, don't log an error
+					// always include "Not found" in error message
+					err = new Error("Failed to head key: " + key + ": Not found");
+					err.code = "NoSuchKey";
+				}
+				else {
+					// some other error
 					self.logError('s3', "Failed to head key: " + key + ": " + (err.message || err), err);
 				}
 				callback( err, null );
@@ -277,7 +311,10 @@ module.exports = Class.create({
 				self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? body : null);
 			}
 			
-			callback( null, body );
+			callback( null, body, {
+				mod: Math.floor((new Date(data.LastModified)).getTime() / 1000),
+				len: data.ContentLength
+			} );
 		} );
 	},
 	

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.0.16",
+	"version": "3.0.19",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",

package/storage.js

@@ -226,6 +226,36 @@ module.exports = Class.create({
 		} );
 	},
 	
+	putStreamCustom: function(key, stream, opts, callback) {
+		// store key+stream with engine-specific opts
+		var self = this;
+		
+		if (!this.started) return callback( new Error("Storage has not completed startup.") );
+		key = this.normalizeKey( key );
+		
+		if (!this.isBinaryKey(key)) {
+			return callback( new Error("Stream values are only allowed with keys containing file extensions, e.g. " + key + ".bin") );
+		}
+		
+		// sanity checks
+		if (!stream || !stream.pipe) return callback( new Error("Not a valid stream.") );
+		
+		// invoke engine and track perf
+		var pf = this.perf.begin('put');
+		
+		this.engine.putStreamCustom( key, stream, opts, function(err) {
+			// put complete
+			var elapsed = pf.end();
+			
+			if (!err) self.logTransaction('put', key, {
+				elapsed_ms: elapsed
+			});
+			
+			callback(err);
+			if (!err) self.emit('putStream', key);
+		} );
+	},
+	
 	putMulti: function(records, callback) {
 		// put multiple records at once, given object of keys and values
 		var self = this;
@@ -324,7 +354,7 @@ module.exports = Class.create({
 		// invoke engine and track perf
 		var pf = this.perf.begin('get');
 		
-		this.engine.get( key, function(err, value) {
+		this.engine.get( key, function(err, value, info) {
 			// get complete
 			var elapsed = pf.end();
 			
@@ -339,8 +369,8 @@ module.exports = Class.create({
 				elapsed_ms: elapsed
 			});
 			
-			callback(null, value);
-			if (!err) self.emit('get', key, value);
+			callback(null, value, info);
+			if (!err) self.emit('get', key, value, info);
 		} );
 	},