pelias-schema 6.7.0 → 7.0.0

package/mappings/partial/admin.json

@@ -8,8 +8,7 @@
       "type": "text",
       "analyzer": "peliasIndexOneEdgeGram",
       "search_analyzer": "peliasAdmin",
-      "similarity": "peliasDefaultSimilarity",
-      "doc_values": false
+      "similarity": "peliasDefaultSimilarity"
     }
   }
 }

package/mappings/partial/countryAbbreviation.json

@@ -8,8 +8,7 @@
       "type": "text",
       "analyzer": "peliasIndexCountryAbbreviationOneEdgeGram",
       "search_analyzer": "peliasQuery",
-      "similarity": "peliasDefaultSimilarity",
-      "doc_values": false
+      "similarity": "peliasDefaultSimilarity"
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pelias-schema",
-  "version": "6.7.0",
+  "version": "7.0.0",
   "author": "pelias",
   "description": "Elasticsearch schema files and tooling for Pelias",
   "homepage": "https://github.com/pelias/schema",
@@ -11,7 +11,6 @@
     "integration": "./bin/integration",
     "create_index": "./bin/create_index",
     "drop_index": "node scripts/drop_index",
-    "reset_type": "node scripts/reset_type",
     "update_settings": "node scripts/update_settings"
   },
   "repository": {
@@ -34,15 +33,16 @@
     "elasticsearch": "^16.0.0",
     "glob": "^7.1.6",
     "lodash": "^4.17.15",
-    "pelias-config": "^4.5.0",
+    "pelias-config": "^6.0.0",
     "pelias-logger": "^1.3.0",
     "semver": "^7.1.1"
   },
   "devDependencies": {
+    "async": "^3.2.5",
     "difflet": "^1.0.1",
-    "elastictest": "^3.0.0",
     "iso3166-1": "^0.5.1",
     "proxyquire": "^2.0.0",
+    "randomstring": "^1.3.0",
     "tap-spec": "^5.0.0",
     "tape": "^5.0.0"
   },

package/scripts/check_plugins.js

@@ -10,15 +10,6 @@ const required = ['analysis-icu'];
 // list of failures
 let failures = [];
 
-// returns the appropriate plugin name for the configured Elasticsearch version
-function elasticsearchPluginUtility() {
-  if (config.esclient.apiVersion === '2.4') {
-    return 'plugin';
-  } else {
-    return 'elasticsearch-plugin';
-  }
-}
-
 cli.header("checking elasticsearch plugins");
 client.nodes.info(null, (err, res) => {
 
@@ -76,7 +67,7 @@ client.nodes.info(null, (err, res) => {
     failures.forEach(failure => {
       console.error( `\nyou can install the missing packages on '${failure.node.name}' [${failure.node.ip}] with the following command(s):\n` );
       failure.plugins.forEach(plugin => {
-        console.error( colors.green( `sudo ${failure.node.settings.path.home}/bin/${elasticsearchPluginUtility()} install ${plugin}`) );
+        console.error( colors.green( `sudo ${failure.node.settings.path.home}/bin/elasticsearch-plugin install ${plugin}`) );
       });
     });
     console.error( colors.white("\nnote:") + "some plugins may require you to restart elasticsearch.\n");

package/scripts/create_index.js

@@ -30,7 +30,6 @@ const indexName = config.schema.indexName;
 const req = {
   index: indexName,
   body: schema,
-  include_type_name: false
 };
 
 client.indices.create(req, (err, res) => {

package/scripts/list_analyzers.js

@@ -1,7 +1,6 @@
 const _ = require('lodash')
 const colors = require('colors')
 const cli = require('./cli');
-const config = require('pelias-config').generate();
 const schema = require('../schema');
 const DEFAULT_ANALYZER = 'standard';
 const NOT_APPLICABLE_ANALYZER = 'n/a';
@@ -54,7 +53,7 @@ const error = function(vals) {
 }
 
 // parse mapping
-const mapping = schema.mappings[config.schema.typeName];
+const mapping = schema.mappings;
 const dynamic = mapping.dynamic_templates.map(t => _.first(_.map(t, v => v)));
 
 // process and single mapping property (recursively)

package/scripts/output_mapping.js

@@ -1,21 +1,4 @@
-var config = require('pelias-config').generate();
-var es = require('elasticsearch');
-var client = new es.Client(config.esclient);
 var schema = require('../schema');
 
-var _index = ( process.argv.length > 3 ) ? process.argv[3] : config.schema.indexName;
-var _type = ( process.argv.length > 2 ) ? process.argv[2] : null; // get type from cli args
-
-// print out mapping for just one type
-if ( _type ) {
-  var mapping = schema.mappings[_type];
-  if( !mapping ){
-    console.error( 'could not find a mapping in the schema file for', _index+'/'+_type );
-    process.exit(1);
-  }
-  console.log( JSON.stringify( mapping, null, 2 ) );
-//print out the entire schema mapping
-} else {
-  console.log( JSON.stringify( schema, null, 2 ) );
-}
+console.log( JSON.stringify( schema.mappings, null, 2 ) );
 

package/settings.js

@@ -226,7 +226,7 @@ function generate(){
           "only_on_same_position": "true"
         },
         "peliasOneEdgeGramFilter": {
-          "type" : "edgeNGram",
+          "type" : "edge_ngram",
           "min_gram" : 1,
           "max_gram" : 24
         },

package/test/configValidation.js

@@ -19,7 +19,7 @@ module.exports.tests.interface = function(test, common) {
 
   test('config without schema.indexName should throw error', function(t) {
     var config = {
-      schema: { typeName: 'example_type' },
+      schema: {},
       esclient: {}
     };
 
@@ -30,25 +30,11 @@ module.exports.tests.interface = function(test, common) {
 
   });
 
-  test('config without schema.typeName should throw error', function (t) {
-    var config = {
-      schema: { indexName: 'example_index' },
-      esclient: {}
-    };
-
-    t.throws(function () {
-      configValidation.validate(config);
-    }, /"schema.typeName" is required/, 'schema.typeName should exist');
-    t.end();
-
-  });
-
   test('config with non-string schema.indexName should throw error', function(t) {
     [null, 17, {}, [], false].forEach((value) => {
       var config = {
         schema: {
           indexName: value,
-          typeName: 'example_type'
         },
         esclient: {}
       };
@@ -63,32 +49,11 @@ module.exports.tests.interface = function(test, common) {
 
   });
 
-  test('config with non-string schema.typeName should throw error', function (t) {
-    [null, 17, {}, [], false].forEach((value) => {
-      var config = {
-        schema: {
-          indexName: 'example_index',
-          typeName: value
-        },
-        esclient: {}
-      };
-
-      t.throws(function () {
-        configValidation.validate(config);
-      }, /"schema.typeName" must be a string/, 'schema.typeName should be a string');
-
-    });
-
-    t.end();
-
-  });
-
   test('config with non-object esclient should throw error', function(t) {
     [null, 17, [], 'string', true].forEach((value) => {
       var config = {
         schema: {
           indexName: 'example_index',
-          typeName: 'example_type'
         },
         esclient: value
       };
@@ -107,7 +72,6 @@ module.exports.tests.interface = function(test, common) {
     var config = {
       schema: {
         indexName: 'example_index',
-        typeName: 'example_type'
       },
       esclient: {}
     };

package/test/elastictest/Suite.js

@@ -0,0 +1,103 @@
+const _ = require('lodash')
+const randomstring = require('randomstring')
+const elasticsearch = require('elasticsearch')
+const async = require('async')
+
+function Suite (clientOpts, props) {
+  this.actions = []
+  this.asserts = []
+  this.client = null
+  this.clientOpts = !!clientOpts && (clone(clientOpts) || {
+    host: 'localhost:9200',
+    keepAlive: true
+  })
+  this.props = props || {}
+  if (!_.has(this, 'props.index')) {
+    this.props.index = 'testindex-' + randomstring.generate(7).toLowerCase()
+  }
+}
+
+Suite.prototype.action = function (action) {
+  this.actions.push(action)
+}
+
+Suite.prototype.assert = function (assert) {
+  this.asserts.push(assert)
+}
+
+Suite.prototype.start = function (cb) {
+  this.client = new elasticsearch.Client(this.clientOpts)
+  cb()
+}
+
+Suite.prototype.create = function (cb) {
+  const cmd = _.merge(this.props.create, { index: this.props.index })
+  if (_.has(this, 'props.schema')) {
+    cmd.body = this.props.schema
+  }
+  this.client.indices.create(cmd, cb)
+}
+
+Suite.prototype.delete = function (cb) {
+  const cmd = _.merge(this.props.delete, { index: this.props.index })
+  this.client.indices.delete(cmd, cb)
+}
+
+Suite.prototype.refresh = function (cb) {
+  const cmd = _.merge(this.props.refresh, { index: this.props.index })
+  this.client.indices.refresh(cmd, cb)
+}
+
+// note: optimize was replaced in version ES@2.0, forcemerge used for later versions:
+// https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html
+Suite.prototype.optimize = function (cb) {
+  try {
+    this.client.indices.forcemerge({ index: this.props.index }, cb)
+  } catch (e) {
+    const cmd = { index: this.props.index, waitForMerge: true, force: true }
+    this.client.indices.optimize(cmd, cb)
+  }
+}
+
+Suite.prototype.waitForStatus = function (status, cb) {
+  const cmd = { waitForStatus: status || 'yellow' }
+  this.client.cluster.health(cmd, cb)
+}
+
+Suite.prototype.run = function (cb) {
+  const self = this
+  self.start(err => {
+    if (err) { throw new Error(err) }
+    self.waitForStatus('yellow', err => {
+      if (err) { throw new Error(err) }
+      self.create(err => {
+        if (err) { throw new Error(err) }
+        async.series(self.actions, err => {
+          if (err) { throw new Error(err) }
+          self.refresh(err => {
+            if (err) { throw new Error(err) }
+            self.optimize(async.series(self.asserts, err => {
+              if (err) { throw new Error(err) }
+              self.delete(err => {
+                if (err) { throw new Error(err) }
+                self.client.close()
+                if (_.isFunction(cb)) {
+                  cb()
+                }
+              })
+            }))
+          })
+        })
+      })
+    })
+  })
+}
+
+// elasticsearch client requires that config objects not be reused.
+// Error: Do not reuse objects to configure the elasticsearch Client class:
+// https://github.com/elasticsearch/elasticsearch-js/issues/33
+function clone (obj) {
+  return JSON.parse(JSON.stringify(obj))
+}
+
+module.exports = Suite

package/test/fixtures/expected.json

@@ -248,7 +248,7 @@
           "only_on_same_position": "true"
         },
         "peliasOneEdgeGramFilter": {
-          "type": "edgeNGram",
+          "type": "edge_ngram",
           "min_gram": 1,
           "max_gram": 24
         },
@@ -2415,8 +2415,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2430,8 +2429,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2453,8 +2451,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2468,8 +2465,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2491,8 +2487,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2506,8 +2501,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2529,8 +2523,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2544,8 +2537,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexCountryAbbreviationOneEdgeGram",
                 "search_analyzer": "peliasQuery",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2567,8 +2559,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2582,8 +2573,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2605,8 +2595,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2620,8 +2609,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2643,8 +2631,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2658,8 +2645,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2681,8 +2667,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2696,8 +2681,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2719,8 +2703,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2734,8 +2717,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2757,8 +2739,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2772,8 +2753,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2795,8 +2775,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2810,8 +2789,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2833,8 +2811,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2848,8 +2825,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2871,8 +2847,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2886,8 +2861,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2909,8 +2883,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2924,8 +2897,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },

package/test/settings.js

@@ -490,7 +490,7 @@ module.exports.tests.peliasOneEdgeGramFilter = function(test, common) {
     var s = settings();
     t.equal(typeof s.analysis.filter.peliasOneEdgeGramFilter, 'object', 'there is a peliasIndexOneEdgeGram filter');
     var filter = s.analysis.filter.peliasOneEdgeGramFilter;
-    t.equal(filter.type, 'edgeNGram');
+    t.equal(filter.type, 'edge_ngram');
     t.equal(filter.min_gram, 1);
     t.equal(filter.max_gram, 24);
     t.end();

package/scripts/reset_type.js

@@ -1,26 +0,0 @@
-var config = require('pelias-config').generate().esclient;
-var es = require('elasticsearch');
-var client = new es.Client(config);
-var schema = require('../schema');
-
-var _index = ( process.argv.length > 3 ) ? process.argv[3] : config.schema.indexName;
-var _type = ( process.argv.length > 2 ) ? process.argv[2] : null; // get type from cli args
-
-if( !_type ){
-  console.error( 'you must provide the target es \'type\' as the first cli argument' );
-  process.exit(1);
-}
-
-var mapping = schema.mappings[_type];
-if( !mapping ){
-  console.error( 'could not find a mapping in the schema file for', _index+'/'+_type );
-  process.exit(1);
-}
-
-client.indices.deleteMapping( { index: _index, type: _type }, function( err, res ){
-  console.log( '[delete mapping]', '\t', _index+'/'+_type, err || '\t', res );
-  client.indices.putMapping( { index: _index, type: _type, body:mapping }, function( err, res ){
-    console.log( '[put mapping]', '\t\t', _index+'/'+_type, err || '\t', res );
-    process.exit( !!err );
-  });
-});