pixl-server-storage 3.1.7 → 3.1.9

package/docs/API.md

@@ -1367,7 +1367,7 @@ storage.getFieldSummary( FIELD_ID, CONFIG, CALLBACK );
 The `getFieldSummary()` method fetches a summary of all word counts for an index field.  This requires a field indexed with the [master list](Indexer.md#master-list) feature enabled.  Then you can fetch a "summary" of the data values, which returns a hash containing all the unique words from the index, and their total counts (occurrences) in the data.  Example use:
 
 ```js
-stotage.getFieldSummary( 'status', config, function(err, values) {
+storage.getFieldSummary( 'status', config, function(err, values) {
 	if (err) throw err;
 	
 	// values should contain a hash with word counts:

package/docs/Indexer.md

@@ -231,6 +231,7 @@ Text filters provide a way to cleanup specific markup languages, leaving only se
 |--------|-------------|
 | `html` | This filter strips all HTML tags, and decodes all HTML entities prior to indexing.  This also works for XML source. |
 | `markdown` | This filter is designed for [Markdown](https://en.wikipedia.org/wiki/Markdown) source text.  It filters out [fenced code blocks](https://help.github.com/articles/creating-and-highlighting-code-blocks/) and also applies the `html` filter as well (you can embed HTML in markdown). |
+| `alphanum` | This filter strips everything except for alphanumerics and underscores.  It's designed to reduce a string to a single indexable value. |
 
 Note that URLs are always shortened so that only the hostname is indexed.  Full URLs are notoriously difficult (and rather useless) to index for searching.  See [Text Cleanup](#text-cleanup) below for details.
 
@@ -784,7 +785,7 @@ Given an array of your sorted record IDs, you can then implement your own pagina
 If you have any fields indexed with the [master list](#master-list) feature enabled, you can fetch a "summary" of the data values using the [getFieldSummary()](API.md#getfieldsummary) method.  This returns a hash containing all the unique words from the index, and their total counts (occurrences) in the data.  Example use:
 
 ```js
-stotage.getFieldSummary( 'status', config, function(err, values) {
+storage.getFieldSummary( 'status', config, function(err, values) {
 	if (err) throw err;
 	
 	// values will contain a hash with word counts:

package/engines/S3.js

@@ -7,6 +7,7 @@ var Component = require("pixl-server/component");
 var Tools = require("pixl-tools");
 var Cache = require("pixl-cache");
 var S3 = require("@aws-sdk/client-s3");
+var { Upload } = require("@aws-sdk/lib-storage");
 var { NodeHttpHandler } = require("@aws-sdk/node-http-handler");
 var streamToBuffer = require("fast-stream-to-buffer");
 
@@ -158,11 +159,11 @@ module.exports = Class.create({
 					self.cache.set( orig_key, params.Body, { date: Tools.timeNow(true) } );
 				}
 				
-				if (callback) callback(null, data);
+				if (callback) process.nextTick( function() { callback(null, data); });
 			} )
 			.catch( function(err) {
 				self.logError('s3', "Failed to store object: " + key + ": " + (err.message || err), err);
-				if (callback) callback(err);
+				if (callback) process.nextTick( function() { callback(err); });
 			} );
 	},
 	
@@ -178,14 +179,19 @@ module.exports = Class.create({
 		
 		this.logDebug(9, "Storing S3 Binary Stream: " + key);
 		
-		this.s3.send( new S3.PutObjectCommand(params) )
+		var upload = new Upload({
+			client: this.s3,
+			params: params
+		});
+		
+		upload.done()
 			.then( function(data) {
 				self.logDebug(9, "Stream store complete: " + key);
-				if (callback) callback(null, data);
+				if (callback) process.nextTick( function() { callback(null, data); });
 			} )
 			.catch( function(err) {
 				self.logError('s3', "Failed to store stream: " + key + ": " + (err.message || err), err);
-				if (callback) callback(err);
+				if (callback) process.nextTick( function() { callback(err, null); });
 			} );
 	},
 	
@@ -202,14 +208,19 @@ module.exports = Class.create({
 		
 		this.logDebug(9, "Storing S3 Binary Stream: " + key);
 		
-		this.s3.send( new S3.PutObjectCommand(params) )
+		var upload = new Upload({
+			client: this.s3,
+			params: params
+		});
+		
+		upload.done()
 			.then( function(data) {
 				self.logDebug(9, "Stream store complete: " + key);
-				if (callback) callback(null, data);
+				if (callback) process.nextTick( function() { callback(null, data); });
 			} )
 			.catch( function(err) {
 				self.logError('s3', "Failed to store stream: " + key + ": " + (err.message || err), err);
-				if (callback) callback(err);
+				if (callback) process.nextTick( function() { callback(err, null); });
 			} );
 	},
 	
@@ -241,9 +252,11 @@ module.exports = Class.create({
 			.then( function(data) {
 				self.logDebug(9, "Head complete: " + key);
 				
-				callback( null, {
-					mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
-					len: data.ContentLength
+				process.nextTick( function() {
+					callback( null, {
+						mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
+						len: data.ContentLength
+					} );
 				} );
 			} )
 			.catch( function(err) {
@@ -257,7 +270,7 @@ module.exports = Class.create({
 					// some other error
 					self.logError('s3', "Failed to head key: " + key + ": " + (err.message || err), err);
 				}
-				callback( err );
+				process.nextTick( function() { callback( err ); } );
 			} );
 	},
 	
@@ -337,7 +350,7 @@ module.exports = Class.create({
 					// some other error
 					self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
 				}
-				callback( err );
+				process.nextTick( function() { callback( err ); } );
 			} );
 	},
 	
@@ -366,9 +379,11 @@ module.exports = Class.create({
 					self.logDebug(9, "S3 stream download closed: " + key);
 				} );
 				
-				callback( null, download, {
-					mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
-					len: data.ContentLength
+				process.nextTick( function() {
+					callback( null, download, {
+						mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
+						len: data.ContentLength
+					} );
 				} );
 			})
 			.catch( function(err) {
@@ -382,7 +397,7 @@ module.exports = Class.create({
 					// some other error
 					self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
 				}
-				callback( err );
+				process.nextTick( function() { callback( err ); } );
 			});
 	},
 	
@@ -425,10 +440,12 @@ module.exports = Class.create({
 					len = parseInt( RegExp.$1 );
 				}
 				
-				callback( null, download, {
-					mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
-					len: len,
-					cr: data.ContentRange
+				process.nextTick( function() { 
+					callback( null, download, {
+						mod: Math.floor( (new Date(data.LastModified)).getTime() / 1000 ),
+						len: len,
+						cr: data.ContentRange
+					} );
 				} );
 			})
 			.catch( function(err) {
@@ -442,7 +459,7 @@ module.exports = Class.create({
 					// some other error
 					self.logError('s3', "Failed to fetch key: " + key + ": " + (err.message || err), err);
 				}
-				callback( err );
+				process.nextTick( function() { callback( err ); } );
 			});
 	},
 	
@@ -466,11 +483,11 @@ module.exports = Class.create({
 					self.cache.delete(orig_key);
 				}
 				
-				if (callback) callback(null, data);
+				if (callback) process.nextTick( function() { callback(null, data); } );
 			} )
 			.catch( function(err) {
 				self.logError('s3', "Failed to delete object: " + key + ": " + (err.message || err), err);
-				if (callback) callback(err);
+				if (callback) process.nextTick( function() { callback(err); } );
 			} );
 	},
 	

package/indexer.js

@@ -239,7 +239,7 @@ module.exports = Class.create({
 							if (checksum != old_data.checksum) {
 								// must reindex
 								state.changed[ def.id ] = 1;
-								self.updateIndex( old_data, data, def, state, callback );
+								self.updateIndex( old_data, data, ''+value, def, state, callback );
 							}
 							else {
 								// data not changed, no action required
@@ -250,7 +250,7 @@ module.exports = Class.create({
 						else {
 							// index doesn't exist for this record, create immediately
 							state.changed[ def.id ] = 1;
-							self.writeIndex( data, def, state, callback );
+							self.writeIndex( data, ''+value, def, state, callback );
 						}
 					}, // iterator
 					function(err) {
@@ -454,7 +454,7 @@ module.exports = Class.create({
 		} ); // lock
 	},
 	
-	writeIndex: function(data, def, state, callback) {
+	writeIndex: function(data, raw_value, def, state, callback) {
 		// create or update single field index
 		var self = this;
 		var words = data.words;
@@ -507,7 +507,8 @@ module.exports = Class.create({
 					def: def,
 					group: group,
 					base_path: base_path,
-					word_hash: word_hash
+					word_hash: word_hash,
+					raw_value: raw_value
 				});
 			} // master_list
 			
@@ -585,6 +586,11 @@ module.exports = Class.create({
 				for (var word in word_hash) {
 					if (!summary.values[word]) summary.values[word] = 0;
 					summary.values[word]++;
+					
+					if (task.def.master_labels) {
+						if (!summary.labels) summary.labels = {};
+						summary.labels[word] = task.raw_value;
+					}
 				} // foreach word
 				
 				// save summary back to storage
@@ -737,7 +743,10 @@ module.exports = Class.create({
 				
 				for (var word in word_hash) {
 					if (summary.values[word]) summary.values[word]--;
-					if (!summary.values[word]) delete summary.values[word];
+					if (!summary.values[word]) {
+						delete summary.values[word];
+						if (task.def.master_labels && summary.labels) delete summary.labels[word];
+					}
 				} // foreach word
 				
 				// save summary back to storage
@@ -765,7 +774,7 @@ module.exports = Class.create({
 		} ); // lock
 	},
 	
-	updateIndex: function(old_data, new_data, def, state, callback) {
+	updateIndex: function(old_data, new_data, raw_value, def, state, callback) {
 		// efficiently update single field index
 		var self = this;
 		var old_words = old_data.words;
@@ -825,6 +834,12 @@ module.exports = Class.create({
 			callback: callback || null
 		};
 		
+		if (!group.count) {
+			this.logDebug(9, "Actually, nothing changed in index: " + def.id + " for record: " + state.id + ", skipping updateIndex");
+			if (callback) callback();
+			return;
+		}
+		
 		// lock index for this
 		self.lock( base_path, true, function() {
 			// update master list if applicable
@@ -838,7 +853,8 @@ module.exports = Class.create({
 						def: def,
 						group: group,
 						base_path: base_path,
-						word_hash: added_words
+						word_hash: added_words,
+						raw_value: raw_value
 					});
 				}
 				if (Tools.numKeys(removed_words) > 0) {
@@ -954,6 +970,11 @@ module.exports = Class.create({
 		return he.decode( value.replace(/<.+?>/g, '') );
 	},
 	
+	filterWords_alphanum: function(value) {
+		// filter out everything except alphanum + underscore
+		return value.replace(/\W+/g, '_').replace(/_+/g, '_');
+	},
+	
 	getWordList: function(value, def, config) {
 		// clean and filter text down to list of alphanumeric words
 		// return array of clean words

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.1.7",
+	"version": "3.1.9",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",
@@ -23,6 +23,8 @@
 	],
 	"dependencies": {
 		"@aws-sdk/client-s3": "3.186.0",
+		"@aws-sdk/lib-storage": "3.186.0",
+		"@aws-sdk/node-http-handler": "3.186.0",
 		"async": "2.6.4",
 		"fast-stream-to-buffer": "1.0.0",
 		"he": "1.1.1",