pelias-schema 10.0.0 → 10.1.0

package/configValidation.js

@@ -3,11 +3,15 @@ const Joi = require('@hapi/joi');
 // Schema Configuration
 // schema.indexName: populated by defaults if not overridden
 // schema.icuTokenizer: boolean, optional, defaults to false
+// schema.unstoredFields: array of field paths to index but omit from _source, optional
+// schema.excludedFields: array of field paths to remove from the mapping, optional
 // esclient: object, validation performed by elasticsearch module
 const schema = Joi.object().required().keys({
   schema: Joi.object().required().keys({
     indexName: Joi.string().required(),
-    icuTokenizer: Joi.boolean().optional()
+    icuTokenizer: Joi.boolean().optional(),
+    unstoredFields: Joi.array().items(Joi.string()).optional(),
+    excludedFields: Joi.array().items(Joi.string()).optional()
   }),
   esclient: Joi.object().required()
 }).unknown(true);

package/mappings/configureFields.js

@@ -0,0 +1,28 @@
+const _ = require('lodash');
+
+// convert a dotted field path such as 'parent.county_a' into its location in a mapping
+const mappingPath = (field) => _.flatMap(field.split('.'), (part) => ['properties', part]);
+
+/**
+ * apply field configuration from `schema` section of pelias config to a mapping
+ *
+ * unstoredFields: fields that are indexed but omitted from _source
+ * excludedFields: fields removed from the mapping entirely
+ */
+function configureFields(mapping, schemaConfig) {
+  const unstored = _.get(schemaConfig, 'unstoredFields', []);
+  const excluded = _.get(schemaConfig, 'excludedFields', []);
+
+  const configured = _.cloneDeep(mapping);
+
+  if (!_.isEmpty(unstored)) {
+    const excludes = _.get(configured, '_source.excludes', []);
+    _.set(configured, '_source.excludes', _.union(excludes, unstored));
+  }
+
+  excluded.forEach((field) => _.unset(configured, mappingPath(field)));
+
+  return configured;
+}
+
+module.exports = configureFields;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pelias-schema",
-  "version": "10.0.0",
+  "version": "10.1.0",
   "author": "pelias",
   "description": "Elasticsearch schema files and tooling for Pelias",
   "homepage": "https://github.com/pelias/schema",

package/schema.js

@@ -1,6 +1,9 @@
+const config = require('pelias-config').generate();
+const configureFields = require('./mappings/configureFields');
+
 const schema = {
   settings: require('./settings')(),
-  mappings: require('./mappings/document'),
+  mappings: configureFields(require('./mappings/document'), config.schema),
 };
 
 module.exports = schema;

package/test/configValidation.js

@@ -84,6 +84,43 @@ module.exports.tests.interface = function(test, common) {
 
   });
 
+  test('config with array of strings for unstoredFields and excludedFields should not throw error', function(t) {
+    var config = {
+      schema: {
+        indexName: 'example_index',
+        unstoredFields: ['addendum', 'parent.county_a'],
+        excludedFields: ['popularity']
+      },
+      esclient: {}
+    };
+
+    t.doesNotThrow(function() {
+      configValidation.validate(config);
+    }, 'no error should have been thrown');
+
+    t.end();
+  });
+
+  test('config with non-array unstoredFields or excludedFields should throw error', function(t) {
+    ['unstoredFields', 'excludedFields'].forEach((key) => {
+      [null, 17, 'addendum', {}, true, [17]].forEach((value) => {
+        var config = {
+          schema: {
+            indexName: 'example_index',
+            [key]: value
+          },
+          esclient: {}
+        };
+
+        t.throws(function() {
+          configValidation.validate(config);
+        }, new RegExp(`"schema.${key}(\\[0\\])?" must be`), `${key} must be an array of strings`);
+      });
+    });
+
+    t.end();
+  });
+
 };
 
 module.exports.all = function (tape, common) {

package/test/configureFields.js

@@ -0,0 +1,49 @@
+const configureFields = require('../mappings/configureFields');
+const document = require('../mappings/document');
+
+module.exports.tests = {};
+
+module.exports.tests.unstored = function(test, common) {
+  test('no config leaves mapping unchanged', function(t) {
+    t.deepEqual(configureFields(document, undefined), document);
+    t.deepEqual(configureFields(document, {}), document);
+    t.end();
+  });
+
+  test('unstored fields are appended to _source excludes', function(t) {
+    const mapping = configureFields(document, { unstoredFields: ['addendum', 'parent.*_a', 'shape'] });
+    t.deepEqual(mapping._source.excludes, ['shape', 'phrase', 'addendum', 'parent.*_a']);
+    t.deepEqual(mapping.properties, document.properties, 'properties unchanged');
+    t.end();
+  });
+};
+
+module.exports.tests.excluded = function(test, common) {
+  test('excluded fields are removed from mapping', function(t) {
+    const mapping = configureFields(document, { excludedFields: ['popularity', 'parent.county_a', 'not_a_field'] });
+    t.false(mapping.properties.popularity, 'top level field removed');
+    t.false(mapping.properties.parent.properties.county_a, 'nested field removed');
+    t.true(mapping.properties.parent.properties.county, 'sibling field kept');
+    t.true(mapping.properties.population, 'other fields kept');
+    t.end();
+  });
+
+  test('original mapping is not modified', function(t) {
+    configureFields(document, { unstoredFields: ['addendum'], excludedFields: ['popularity', 'parent.county_a'] });
+    t.deepEqual(document._source.excludes, ['shape', 'phrase']);
+    t.true(document.properties.popularity);
+    t.true(document.properties.parent.properties.county_a);
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('configureFields: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

package/test/run.js

@@ -11,6 +11,7 @@ var common = {
 var tests = [
   require('./compile.js'),
   require('./document.js'),
+  require('./configureFields.js'),
   require('./partial-centroid.js'),
   require('./partial-admin.js'),
   require('./partial-keyword.js'),