pixl-server-storage 3.1.8 → 3.1.9

package/docs/API.md

@@ -1367,7 +1367,7 @@ storage.getFieldSummary( FIELD_ID, CONFIG, CALLBACK );
 The `getFieldSummary()` method fetches a summary of all word counts for an index field.  This requires a field indexed with the [master list](Indexer.md#master-list) feature enabled.  Then you can fetch a "summary" of the data values, which returns a hash containing all the unique words from the index, and their total counts (occurrences) in the data.  Example use:
 
 ```js
-stotage.getFieldSummary( 'status', config, function(err, values) {
+storage.getFieldSummary( 'status', config, function(err, values) {
 	if (err) throw err;
 	
 	// values should contain a hash with word counts:

package/docs/Indexer.md

@@ -231,6 +231,7 @@ Text filters provide a way to cleanup specific markup languages, leaving only se
 |--------|-------------|
 | `html` | This filter strips all HTML tags, and decodes all HTML entities prior to indexing.  This also works for XML source. |
 | `markdown` | This filter is designed for [Markdown](https://en.wikipedia.org/wiki/Markdown) source text.  It filters out [fenced code blocks](https://help.github.com/articles/creating-and-highlighting-code-blocks/) and also applies the `html` filter as well (you can embed HTML in markdown). |
+| `alphanum` | This filter strips everything except for alphanumerics and underscores.  It's designed to reduce a string to a single indexable value. |
 
 Note that URLs are always shortened so that only the hostname is indexed.  Full URLs are notoriously difficult (and rather useless) to index for searching.  See [Text Cleanup](#text-cleanup) below for details.
 
@@ -784,7 +785,7 @@ Given an array of your sorted record IDs, you can then implement your own pagina
 If you have any fields indexed with the [master list](#master-list) feature enabled, you can fetch a "summary" of the data values using the [getFieldSummary()](API.md#getfieldsummary) method.  This returns a hash containing all the unique words from the index, and their total counts (occurrences) in the data.  Example use:
 
 ```js
-stotage.getFieldSummary( 'status', config, function(err, values) {
+storage.getFieldSummary( 'status', config, function(err, values) {
 	if (err) throw err;
 	
 	// values will contain a hash with word counts:

package/indexer.js

@@ -239,7 +239,7 @@ module.exports = Class.create({
 							if (checksum != old_data.checksum) {
 								// must reindex
 								state.changed[ def.id ] = 1;
-								self.updateIndex( old_data, data, def, state, callback );
+								self.updateIndex( old_data, data, ''+value, def, state, callback );
 							}
 							else {
 								// data not changed, no action required
@@ -250,7 +250,7 @@ module.exports = Class.create({
 						else {
 							// index doesn't exist for this record, create immediately
 							state.changed[ def.id ] = 1;
-							self.writeIndex( data, def, state, callback );
+							self.writeIndex( data, ''+value, def, state, callback );
 						}
 					}, // iterator
 					function(err) {
@@ -454,7 +454,7 @@ module.exports = Class.create({
 		} ); // lock
 	},
 	
-	writeIndex: function(data, def, state, callback) {
+	writeIndex: function(data, raw_value, def, state, callback) {
 		// create or update single field index
 		var self = this;
 		var words = data.words;
@@ -507,7 +507,8 @@ module.exports = Class.create({
 					def: def,
 					group: group,
 					base_path: base_path,
-					word_hash: word_hash
+					word_hash: word_hash,
+					raw_value: raw_value
 				});
 			} // master_list
 			
@@ -585,6 +586,11 @@ module.exports = Class.create({
 				for (var word in word_hash) {
 					if (!summary.values[word]) summary.values[word] = 0;
 					summary.values[word]++;
+					
+					if (task.def.master_labels) {
+						if (!summary.labels) summary.labels = {};
+						summary.labels[word] = task.raw_value;
+					}
 				} // foreach word
 				
 				// save summary back to storage
@@ -737,7 +743,10 @@ module.exports = Class.create({
 				
 				for (var word in word_hash) {
 					if (summary.values[word]) summary.values[word]--;
-					if (!summary.values[word]) delete summary.values[word];
+					if (!summary.values[word]) {
+						delete summary.values[word];
+						if (task.def.master_labels && summary.labels) delete summary.labels[word];
+					}
 				} // foreach word
 				
 				// save summary back to storage
@@ -765,7 +774,7 @@ module.exports = Class.create({
 		} ); // lock
 	},
 	
-	updateIndex: function(old_data, new_data, def, state, callback) {
+	updateIndex: function(old_data, new_data, raw_value, def, state, callback) {
 		// efficiently update single field index
 		var self = this;
 		var old_words = old_data.words;
@@ -825,6 +834,12 @@ module.exports = Class.create({
 			callback: callback || null
 		};
 		
+		if (!group.count) {
+			this.logDebug(9, "Actually, nothing changed in index: " + def.id + " for record: " + state.id + ", skipping updateIndex");
+			if (callback) callback();
+			return;
+		}
+		
 		// lock index for this
 		self.lock( base_path, true, function() {
 			// update master list if applicable
@@ -838,7 +853,8 @@ module.exports = Class.create({
 						def: def,
 						group: group,
 						base_path: base_path,
-						word_hash: added_words
+						word_hash: added_words,
+						raw_value: raw_value
 					});
 				}
 				if (Tools.numKeys(removed_words) > 0) {
@@ -954,6 +970,11 @@ module.exports = Class.create({
 		return he.decode( value.replace(/<.+?>/g, '') );
 	},
 	
+	filterWords_alphanum: function(value) {
+		// filter out everything except alphanum + underscore
+		return value.replace(/\W+/g, '_').replace(/_+/g, '_');
+	},
+	
 	getWordList: function(value, def, config) {
 		// clean and filter text down to list of alphanumeric words
 		// return array of clean words

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "pixl-server-storage",
-	"version": "3.1.8",
+	"version": "3.1.9",
 	"description": "A key/value/list storage component for the pixl-server framework.",
 	"author": "Joseph Huckaby <jhuckaby@gmail.com>",
 	"homepage": "https://github.com/jhuckaby/pixl-server-storage",