pelias-schema 6.8.0 → 7.1.0

package/integration/source_layer_sourceid_filtering.js

@@ -15,28 +15,28 @@ module.exports.tests.source_filter = function(test, common){
     // index some docs
     suite.action( function( done ){
       suite.client.index({
-        index: suite.props.index, type: config.schema.typeName,
+        index: suite.props.index,
         id: '1', body: { source: 'osm', layer: 'node', source_id: 'dataset/1' }
       }, done );
     });
 
     suite.action( function( done ){
       suite.client.index({
-        index: suite.props.index, type: config.schema.typeName,
+        index: suite.props.index, 
         id: '2', body: { source: 'osm', layer: 'address', source_id: 'dataset/2' }
       }, done );
     });
 
     suite.action( function( done ){
       suite.client.index({
-        index: suite.props.index, type: config.schema.typeName,
+        index: suite.props.index, 
         id: '3', body: { source: 'geonames', layer: 'address', source_id: 'dataset/1' }
       }, done );
     });
 
     suite.action( function( done ){
       suite.client.index({
-        index: suite.props.index, type: config.schema.typeName,
+        index: suite.props.index,
         id: '4', body: { source: 'foo bar baz' }
       }, done );
     });
@@ -45,7 +45,6 @@ module.exports.tests.source_filter = function(test, common){
     suite.assert( function( done ){
       suite.client.search({
         index: suite.props.index,
-        type: config.schema.typeName,
         body: { query: {
           term: {
             source: 'osm'
@@ -61,7 +60,6 @@ module.exports.tests.source_filter = function(test, common){
     suite.assert( function( done ){
       suite.client.search({
         index: suite.props.index,
-        type: config.schema.typeName,
         body: { query: {
           term: {
             layer: 'address'
@@ -77,7 +75,6 @@ module.exports.tests.source_filter = function(test, common){
     suite.assert( function( done ){
       suite.client.search({
         index: suite.props.index,
-        type: config.schema.typeName,
         body: { query: {
           term: {
             source_id: 'dataset/1'
@@ -93,7 +90,6 @@ module.exports.tests.source_filter = function(test, common){
     suite.assert( function( done ){
       suite.client.search({
         index: suite.props.index,
-        type: config.schema.typeName,
         body: { query: { bool: { must: [
           { term: { source: 'osm' } },
           { term: { source_id: 'dataset/1' } }
@@ -108,7 +104,6 @@ module.exports.tests.source_filter = function(test, common){
     suite.assert( function( done ){
       suite.client.search({
         index: suite.props.index,
-        type: config.schema.typeName,
         body: { query: {
           term: {
             source: 'OSM'
@@ -124,7 +119,6 @@ module.exports.tests.source_filter = function(test, common){
     suite.assert( function( done ){
       suite.client.search({
         index: suite.props.index,
-        type: config.schema.typeName,
         body: { query: {
           term: {
             source: 'foo'
@@ -140,7 +134,6 @@ module.exports.tests.source_filter = function(test, common){
     suite.assert( function( done ){
       suite.client.search({
         index: suite.props.index,
-        type: config.schema.typeName,
         body: { query: {
           term: {
             source: 'foo bar baz'

package/mappings/partial/admin.json

@@ -8,8 +8,7 @@
       "type": "text",
       "analyzer": "peliasIndexOneEdgeGram",
       "search_analyzer": "peliasAdmin",
-      "similarity": "peliasDefaultSimilarity",
-      "doc_values": false
+      "similarity": "peliasDefaultSimilarity"
     }
   }
 }

package/mappings/partial/countryAbbreviation.json

@@ -8,8 +8,7 @@
       "type": "text",
       "analyzer": "peliasIndexCountryAbbreviationOneEdgeGram",
       "search_analyzer": "peliasQuery",
-      "similarity": "peliasDefaultSimilarity",
-      "doc_values": false
+      "similarity": "peliasDefaultSimilarity"
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pelias-schema",
-  "version": "6.8.0",
+  "version": "7.1.0",
   "author": "pelias",
   "description": "Elasticsearch schema files and tooling for Pelias",
   "homepage": "https://github.com/pelias/schema",
@@ -11,7 +11,6 @@
     "integration": "./bin/integration",
     "create_index": "./bin/create_index",
     "drop_index": "node scripts/drop_index",
-    "reset_type": "node scripts/reset_type",
     "update_settings": "node scripts/update_settings"
   },
   "repository": {
@@ -34,7 +33,7 @@
     "elasticsearch": "^16.0.0",
     "glob": "^7.1.6",
     "lodash": "^4.17.15",
-    "pelias-config": "^4.5.0",
+    "pelias-config": "^6.0.0",
     "pelias-logger": "^1.3.0",
     "semver": "^7.1.1"
   },

package/punctuation.js

@@ -1,23 +1,18 @@
 // These characters will be removed from ngrams/shingles
 // @see: org/apache/lucene/analysis/cn/smart/stopwords.txt
 
-module.exports.all = [
-  ".","`","‘","-","_","=","?","'","|","\"","(",")","{","}","[","]","<",">","*",
-  "#","&","^","$","@","!","~",":",";","+","《","》","—","－","，","。",
-  "、", "：","；","！","·","？","„","“","”","）","（","【","】","［","］","●"
+const all = [
+  ".","`","‘","’","‛","-","_","=","?","'","|","\"","(",")","{","}","[","]","<",">","*",
+  "#","&","^","$","@","!","~",":",";","+","《","》","—","－","，","。","‹","›","⹂","〝","〞",
+  "、", "：","；","！","·","？","„","“","”","‟","）","（","【","】","［","］","●","«","»"
 ];
 
-module.exports.allowed = [
+const allowed = [
   "-", // allow hypens
   "&"  // allow ampersands
 ];
 
-module.exports.blacklist = module.exports.all.slice();
-
 // remove alowed chars from blacklist
-module.exports.allowed.forEach(function(item){
-  var index = module.exports.blacklist.indexOf(item);
-  if( index > -1 ){
-    module.exports.blacklist.splice(index, 1);
-  }
-});
+const blacklist = all.filter(s => !allowed.includes(s));
+
+module.exports = { all, allowed, blacklist };

package/scripts/check_plugins.js

@@ -10,15 +10,6 @@ const required = ['analysis-icu'];
 // list of failures
 let failures = [];
 
-// returns the appropriate plugin name for the configured Elasticsearch version
-function elasticsearchPluginUtility() {
-  if (config.esclient.apiVersion === '2.4') {
-    return 'plugin';
-  } else {
-    return 'elasticsearch-plugin';
-  }
-}
-
 cli.header("checking elasticsearch plugins");
 client.nodes.info(null, (err, res) => {
 
@@ -76,7 +67,7 @@ client.nodes.info(null, (err, res) => {
     failures.forEach(failure => {
       console.error( `\nyou can install the missing packages on '${failure.node.name}' [${failure.node.ip}] with the following command(s):\n` );
       failure.plugins.forEach(plugin => {
-        console.error( colors.green( `sudo ${failure.node.settings.path.home}/bin/${elasticsearchPluginUtility()} install ${plugin}`) );
+        console.error( colors.green( `sudo ${failure.node.settings.path.home}/bin/elasticsearch-plugin install ${plugin}`) );
       });
     });
     console.error( colors.white("\nnote:") + "some plugins may require you to restart elasticsearch.\n");

package/scripts/create_index.js

@@ -30,7 +30,6 @@ const indexName = config.schema.indexName;
 const req = {
   index: indexName,
   body: schema,
-  include_type_name: false
 };
 
 client.indices.create(req, (err, res) => {

package/scripts/list_analyzers.js

@@ -1,7 +1,6 @@
 const _ = require('lodash')
 const colors = require('colors')
 const cli = require('./cli');
-const config = require('pelias-config').generate();
 const schema = require('../schema');
 const DEFAULT_ANALYZER = 'standard';
 const NOT_APPLICABLE_ANALYZER = 'n/a';
@@ -54,7 +53,7 @@ const error = function(vals) {
 }
 
 // parse mapping
-const mapping = schema.mappings[config.schema.typeName];
+const mapping = schema.mappings;
 const dynamic = mapping.dynamic_templates.map(t => _.first(_.map(t, v => v)));
 
 // process and single mapping property (recursively)

package/scripts/output_mapping.js

@@ -1,21 +1,4 @@
-var config = require('pelias-config').generate();
-var es = require('elasticsearch');
-var client = new es.Client(config.esclient);
 var schema = require('../schema');
 
-var _index = ( process.argv.length > 3 ) ? process.argv[3] : config.schema.indexName;
-var _type = ( process.argv.length > 2 ) ? process.argv[2] : null; // get type from cli args
-
-// print out mapping for just one type
-if ( _type ) {
-  var mapping = schema.mappings[_type];
-  if( !mapping ){
-    console.error( 'could not find a mapping in the schema file for', _index+'/'+_type );
-    process.exit(1);
-  }
-  console.log( JSON.stringify( mapping, null, 2 ) );
-//print out the entire schema mapping
-} else {
-  console.log( JSON.stringify( schema, null, 2 ) );
-}
+console.log( JSON.stringify( schema.mappings, null, 2 ) );
 

package/settings.js

@@ -226,7 +226,7 @@ function generate(){
           "only_on_same_position": "true"
         },
         "peliasOneEdgeGramFilter": {
-          "type" : "edgeNGram",
+          "type" : "edge_ngram",
           "min_gram" : 1,
           "max_gram" : 24
         },

package/test/configValidation.js

@@ -19,7 +19,7 @@ module.exports.tests.interface = function(test, common) {
 
   test('config without schema.indexName should throw error', function(t) {
     var config = {
-      schema: { typeName: 'example_type' },
+      schema: {},
       esclient: {}
     };
 
@@ -30,25 +30,11 @@ module.exports.tests.interface = function(test, common) {
 
   });
 
-  test('config without schema.typeName should throw error', function (t) {
-    var config = {
-      schema: { indexName: 'example_index' },
-      esclient: {}
-    };
-
-    t.throws(function () {
-      configValidation.validate(config);
-    }, /"schema.typeName" is required/, 'schema.typeName should exist');
-    t.end();
-
-  });
-
   test('config with non-string schema.indexName should throw error', function(t) {
     [null, 17, {}, [], false].forEach((value) => {
       var config = {
         schema: {
           indexName: value,
-          typeName: 'example_type'
         },
         esclient: {}
       };
@@ -63,32 +49,11 @@ module.exports.tests.interface = function(test, common) {
 
   });
 
-  test('config with non-string schema.typeName should throw error', function (t) {
-    [null, 17, {}, [], false].forEach((value) => {
-      var config = {
-        schema: {
-          indexName: 'example_index',
-          typeName: value
-        },
-        esclient: {}
-      };
-
-      t.throws(function () {
-        configValidation.validate(config);
-      }, /"schema.typeName" must be a string/, 'schema.typeName should be a string');
-
-    });
-
-    t.end();
-
-  });
-
   test('config with non-object esclient should throw error', function(t) {
     [null, 17, [], 'string', true].forEach((value) => {
       var config = {
         schema: {
           indexName: 'example_index',
-          typeName: 'example_type'
         },
         esclient: value
       };
@@ -107,7 +72,6 @@ module.exports.tests.interface = function(test, common) {
     var config = {
       schema: {
         indexName: 'example_index',
-        typeName: 'example_type'
       },
       esclient: {}
     };

package/test/fixtures/expected.json

@@ -248,7 +248,7 @@
           "only_on_same_position": "true"
         },
         "peliasOneEdgeGramFilter": {
-          "type": "edgeNGram",
+          "type": "edge_ngram",
           "min_gram": 1,
           "max_gram": 24
         },
@@ -2277,6 +2277,8 @@
             ".=>",
             "`=>",
             "‘=>",
+            "’=>",
+            "‛=>",
             "_=>",
             "==>",
             "?=>",
@@ -2307,6 +2309,11 @@
             "－=>",
             "，=>",
             "。=>",
+            "‹=>",
+            "›=>",
+            "⹂=>",
+            "〝=>",
+            "〞=>",
             "、=>",
             "：=>",
             "；=>",
@@ -2316,13 +2323,16 @@
             "„=>",
             "“=>",
             "”=>",
+            "‟=>",
             "）=>",
             "（=>",
             "【=>",
             "】=>",
             "［=>",
             "］=>",
-            "●=>"
+            "●=>",
+            "«=>",
+            "»=>"
           ]
         },
         "alphanumeric": {
@@ -2415,8 +2425,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2430,8 +2439,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2453,8 +2461,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2468,8 +2475,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2491,8 +2497,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2506,8 +2511,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2529,8 +2533,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2544,8 +2547,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexCountryAbbreviationOneEdgeGram",
                 "search_analyzer": "peliasQuery",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2567,8 +2569,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2582,8 +2583,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2605,8 +2605,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2620,8 +2619,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2643,8 +2641,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2658,8 +2655,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2681,8 +2677,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2696,8 +2691,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2719,8 +2713,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2734,8 +2727,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2757,8 +2749,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2772,8 +2763,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2795,8 +2785,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2810,8 +2799,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2833,8 +2821,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2848,8 +2835,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2871,8 +2857,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2886,8 +2871,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2909,8 +2893,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -2924,8 +2907,7 @@
                 "type": "text",
                 "analyzer": "peliasIndexOneEdgeGram",
                 "search_analyzer": "peliasAdmin",
-                "similarity": "peliasDefaultSimilarity",
-                "doc_values": false
+                "similarity": "peliasDefaultSimilarity"
               }
             }
           },
@@ -3051,4 +3033,4 @@
     },
     "dynamic": "strict"
   }
-}
+}

package/test/settings.js

@@ -490,7 +490,7 @@ module.exports.tests.peliasOneEdgeGramFilter = function(test, common) {
     var s = settings();
     t.equal(typeof s.analysis.filter.peliasOneEdgeGramFilter, 'object', 'there is a peliasIndexOneEdgeGram filter');
     var filter = s.analysis.filter.peliasOneEdgeGramFilter;
-    t.equal(filter.type, 'edgeNGram');
+    t.equal(filter.type, 'edge_ngram');
     t.equal(filter.min_gram, 1);
     t.equal(filter.max_gram, 24);
     t.end();
@@ -591,7 +591,7 @@ module.exports.tests.punctuationCharFilter = function(test, common) {
     var char_filter = s.analysis.char_filter.punctuation;
     t.equal(char_filter.type, 'mapping');
     t.true(Array.isArray(char_filter.mappings));
-    t.equal(char_filter.mappings.length, 49);
+    t.equal(char_filter.mappings.length, 59);
     t.end();
   });
 };