pixl-server-storage 3.0.18 → 3.0.21

package/README.md

@@ -523,10 +523,12 @@ Note that binary records are **not** cached, as they are generally large.  Only
 
 ## Couchbase
 
-If you want to use [Couchbase](http://www.couchbase.com/nosql-databases/couchbase-server) as a backing store, here is how to do so.  First, you need to manually install the [couchbase](https://www.npmjs.com/package/couchbase) module into your app:
+Please note that as of this writing (April 2022), pixl-server-storage only supports Couchbase Client v2, so you need to force install version `2.6.12` (see instructions below).  Work is underway to support the v3 API, which has many breaking changes.
+
+If you want to use [Couchbase](http://www.couchbase.com/nosql-databases/couchbase-server) as a backing store, here is how to do so.  First, you need to manually install the [couchbase](https://www.npmjs.com/package/couchbase) module into your app, and it **must be v2**:
 
 ```
-npm install --save couchbase
+npm install --save couchbase@2.6.12
 ```
 
 Then configure your storage thusly:

package/engines/Filesystem.js

@@ -315,6 +315,58 @@ module.exports = Class.create({
 		inp.pipe( outp );
 	},
 	
+	putStreamCustom: function(key, inp, opts, callback) {
+		// store key+stream of data to disk
+		var self = this;
+		var file = this.getFilePath(key);
+		
+		this.logDebug(9, "Storing Binary Stream Object: " + key, file);
+		
+		var dir = path.dirname( file );
+		
+		var temp_file = this.tempDir + '/' + path.basename(file) + '.tmp.' + this.tempFileCounter;
+		this.tempFileCounter = (this.tempFileCounter + 1) % 10000000;
+		
+		// create the write stream to temp file
+		var outp = fs.createWriteStream( temp_file, opts || {} );
+		
+		outp.on('error', function(err) {
+			// failed to write file
+			var msg = "Failed to write file: " + key + ": " + temp_file + ": " + err.message;
+			self.logError('file', msg);
+			return callback( new Error(msg), null );
+		} );
+		
+		outp.on('finish', function() {
+			// make sure parent dirs exist, async
+			self._makeDirs( dir, 0o0775, function(err) {
+				if (err) {
+					// failed to create directory
+					var msg = "Failed to create directory: " + key + ": " + dir + ": " + err.message;
+					self.logError('file', msg);
+					return callback( new Error(msg), null );
+				}
+				
+				// rename temp file to final
+				self._renameFile( temp_file, file, function (err) {
+					if (err) {
+						// failed to write file
+						var msg = "Failed to rename file: " + key + ": " + temp_file + ": " + err.message;
+						self.logError('file', msg);
+						return callback( new Error(msg), null );
+					}
+					
+					// all done
+					self.logDebug(9, "Store operation complete: " + key);
+					callback(null, null);
+				} ); // rename
+			} ); // mkdirp
+		} ); // pipe finish
+		
+		// pipe inp to outp
+		inp.pipe( outp );
+	},
+	
 	head: function(key, callback) {
 		// head value given key
 		var self = this;

package/engines/S3.js

@@ -166,6 +166,33 @@ module.exports = Class.create({
 		} );
 	},
 	
+	putStreamCustom: function(key, inp, opts, callback) {
+		// store key+stream of data to S3, inc options
+		var self = this;
+		var orig_key = key;
+		key = this.prepKey(key);
+		
+		var params = {};
+		params.Key = this.extKey(key, orig_key);
+		params.Body = inp;
+		if (opts) Tools.mergeHashInto(params, opts);
+		
+		this.logDebug(9, "Storing S3 Binary Stream: " + key);
+		
+		// double-callback protection (bug in aws-sdk)
+		var done = false;
+		this.s3.upload(params, function(err, data) {
+			if (done) return; else done = true;
+			
+			if (err) {
+				self.logError('s3', "Failed to store stream: " + key + ": " + (err.message || err), err);
+			}
+			else self.logDebug(9, "Stream store complete: " + key);
+			
+			if (callback) callback(err, data);
+		} );
+	},
+	
 	head: function(key, callback) {
 		// head s3 value given key
 		var self = this;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.0.18",
+	"version": "3.0.21",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -22,11 +22,11 @@
 	],
 	"dependencies": {
 		"pixl-server": "^1.0.0",
-		"pixl-class": "^1.0.0",
+		"pixl-class": "^1.0.3",
 		"pixl-tools": "^1.0.24",
 		"pixl-perf": "^1.0.0",
 		"pixl-cache": "^1.0.1",
-		"async": "2.6.0",
+		"async": "2.6.4",
 		"nearley": "2.20.1",
 		"moo": "0.4.3",
 		"porter-stemmer": "0.9.1",

package/storage.js

@@ -226,6 +226,36 @@ module.exports = Class.create({
 		} );
 	},
 	
+	putStreamCustom: function(key, stream, opts, callback) {
+		// store key+stream with engine-specific opts
+		var self = this;
+		
+		if (!this.started) return callback( new Error("Storage has not completed startup.") );
+		key = this.normalizeKey( key );
+		
+		if (!this.isBinaryKey(key)) {
+			return callback( new Error("Stream values are only allowed with keys containing file extensions, e.g. " + key + ".bin") );
+		}
+		
+		// sanity checks
+		if (!stream || !stream.pipe) return callback( new Error("Not a valid stream.") );
+		
+		// invoke engine and track perf
+		var pf = this.perf.begin('put');
+		
+		this.engine.putStreamCustom( key, stream, opts, function(err) {
+			// put complete
+			var elapsed = pf.end();
+			
+			if (!err) self.logTransaction('put', key, {
+				elapsed_ms: elapsed
+			});
+			
+			callback(err);
+			if (!err) self.emit('putStream', key);
+		} );
+	},
+	
 	putMulti: function(records, callback) {
 		// put multiple records at once, given object of keys and values
 		var self = this;

package/test/test-hash.js

@@ -10,7 +10,7 @@ var async = require('async');
 var Tools = require('pixl-tools');
 
 // const BAD_KEYS = Object.getOwnPropertyNames( Object.prototype );
-const BAD_KEYS = ['constructor', '__defineGetter__', '__defineSetter__', 'hasOwnProperty', '__lookupGetter__', '__lookupSetter__', 'isPrototypeOf', 'propertyIsEnumerable', 'toString', 'valueOf', '__proto__', 'toLocaleString'];
+const BAD_KEYS = ['constructor', '__defineGetter__', '__defineSetter__', 'hasOwnProperty', '__lookupGetter__', '__lookupSetter__', 'isPrototypeOf', 'propertyIsEnumerable', 'toString', 'valueOf', 'toLocaleString'];
 
 module.exports = {
 	tests: [