pelias-schema 8.0.0 → 8.2.0

package/test/fixtures/expected.json

@@ -55,7 +55,6 @@
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "peliasOneEdgeGramFilter",
             "unique_only_same_position",
             "notnull",
@@ -74,7 +73,6 @@
             "trim",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "unique_only_same_position",
             "notnull"
           ]
@@ -96,7 +94,6 @@
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "unique_only_same_position",
             "notnull",
             "flatten_graph"
@@ -252,11 +249,6 @@
           "min_gram": 1,
           "max_gram": 24
         },
-        "removeAllZeroNumericPrefix": {
-          "type": "pattern_replace",
-          "pattern": "^(0*)",
-          "replacement": ""
-        },
         "remove_ordinals": {
           "type": "pattern_replace",
           "pattern": "(?i)((^| )((1)st?|(2)nd?|(3)rd?|([4-9])th?)|(([0-9]*)(1[0-9])th?)|(([0-9]*[02-9])((1)st?|(2)nd?|(3)rd?|([04-9])th?))($| ))",

package/test/settings.js

@@ -1,6 +1,7 @@
-var path = require('path'),
+const path = require('path'),
     settings = require('../settings'),
-    fs = require('fs');
+    fs = require('fs'),
+    config = require('pelias-config').generate();
 
 module.exports.tests = {};
 
@@ -48,6 +49,20 @@ module.exports.tests.analysis = function(test, common) {
   });
 };
 
+function mayBeAmpersandMapper() {
+  if (config.schema.icuTokenizer) {
+    return ['ampersand_mapper'];
+  }
+  return [];
+}
+
+function mayBeAmpersandReplacer() {
+  if (config.schema.icuTokenizer) {
+    return ['ampersand_replacer'];
+  }
+  return [];
+}
+
 // -- analyzers --
 
 module.exports.tests.peliasAdminAnalyzer = function(test, common) {
@@ -57,13 +72,13 @@ module.exports.tests.peliasAdminAnalyzer = function(test, common) {
     var analyzer = s.analysis.analyzer.peliasAdmin;
     t.equal(analyzer.type, 'custom', 'custom analyzer');
     t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
-    t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified');
+    t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), 'punctuation', 'nfkc_normalizer'], 'character filters specified');
     t.true(Array.isArray(analyzer.filter), 'filters specified');
     t.end();
   });
   test('peliasAdmin token filters', function (t) {
     var analyzer = settings().analysis.analyzer.peliasAdmin;
-    t.deepEqual(analyzer.filter, [
+    t.deepEqual(analyzer.filter, [...mayBeAmpersandReplacer(),
       "lowercase",
       "trim",
       "synonyms/custom_admin/multiword",
@@ -85,13 +100,14 @@ module.exports.tests.peliasIndexOneEdgeGramAnalyzer = function(test, common) {
     var analyzer = s.analysis.analyzer.peliasIndexOneEdgeGram;
     t.equal(analyzer.type, 'custom', 'custom analyzer');
     t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
-    t.deepEqual(analyzer.char_filter, ["punctuation","nfkc_normalizer"], 'character filters specified');
+    t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), "punctuation","nfkc_normalizer"], 'character filters specified');
     t.true(Array.isArray(analyzer.filter), 'filters specified');
     t.end();
   });
   test('peliasIndexOneEdgeGram token filters', function(t) {
     var analyzer = settings().analysis.analyzer.peliasIndexOneEdgeGram;
     t.deepEqual( analyzer.filter, [
+      ...mayBeAmpersandReplacer(),
       "lowercase",
       "trim",
       "synonyms/custom_name/multiword",
@@ -100,7 +116,6 @@ module.exports.tests.peliasIndexOneEdgeGramAnalyzer = function(test, common) {
       "name_synonyms_multiplexer",
       "icu_folding",
       "remove_ordinals",
-      "removeAllZeroNumericPrefix",
       "peliasOneEdgeGramFilter",
       "unique_only_same_position",
       "notnull",
@@ -117,18 +132,18 @@ module.exports.tests.peliasQueryAnalyzer = function (test, common) {
     var analyzer = s.analysis.analyzer.peliasQuery;
     t.equal(analyzer.type, 'custom', 'custom analyzer');
     t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
-    t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified');
+    t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), 'punctuation', 'nfkc_normalizer'], 'character filters specified');
     t.true(Array.isArray(analyzer.filter), 'filters specified');
     t.end();
   });
   test('peliasQuery token filters', function (t) {
     var analyzer = settings().analysis.analyzer.peliasQuery;
     t.deepEqual(analyzer.filter, [
+      ...mayBeAmpersandReplacer(),
       'lowercase',
       'trim',
       'icu_folding',
       'remove_ordinals',
-      'removeAllZeroNumericPrefix',
       'unique_only_same_position',
       'notnull'
     ]);
@@ -143,13 +158,14 @@ module.exports.tests.peliasPhraseAnalyzer = function(test, common) {
     var analyzer = s.analysis.analyzer.peliasPhrase;
     t.equal(analyzer.type, 'custom', 'custom analyzer');
     t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
-    t.deepEqual(analyzer.char_filter, ["punctuation","nfkc_normalizer"], 'character filters specified');
+    t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), "punctuation", "nfkc_normalizer"], 'character filters specified');
     t.true(Array.isArray(analyzer.filter), 'filters specified');
     t.end();
   });
   test('peliasPhrase token filters', function(t) {
     var analyzer = settings().analysis.analyzer.peliasPhrase;
     t.deepEqual( analyzer.filter, [
+      ...mayBeAmpersandReplacer(),
       "lowercase",
       "trim",
       "remove_duplicate_spaces",
@@ -159,7 +175,6 @@ module.exports.tests.peliasPhraseAnalyzer = function(test, common) {
       "name_synonyms_multiplexer",
       "icu_folding",
       "remove_ordinals",
-      "removeAllZeroNumericPrefix",
       "unique_only_same_position",
       "notnull",
       "flatten_graph"
@@ -236,13 +251,13 @@ module.exports.tests.peliasStreetAnalyzer = function(test, common) {
     var analyzer = s.analysis.analyzer.peliasStreet;
     t.equal(analyzer.type, 'custom', 'custom analyzer');
     t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
-    t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified');
+    t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), 'punctuation', 'nfkc_normalizer'], 'character filters specified');
     t.true(Array.isArray(analyzer.filter), 'filters specified');
     t.end();
   });
   test('peliasStreet token filters', function(t) {
     var analyzer = settings().analysis.analyzer.peliasStreet;
-    t.deepEqual( analyzer.filter, [
+    t.deepEqual( analyzer.filter, [...mayBeAmpersandReplacer(),
       "lowercase",
       "trim",
       "remove_duplicate_spaces",
@@ -266,13 +281,13 @@ module.exports.tests.peliasIndexCountryAbbreviation = function (test, common) {
     var analyzer = s.analysis.analyzer.peliasIndexCountryAbbreviation;
     t.equal(analyzer.type, 'custom', 'custom analyzer');
     t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
-    t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified');
+    t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), 'punctuation', 'nfkc_normalizer'], 'character filters specified');
     t.true(Array.isArray(analyzer.filter), 'filters specified');
     t.end();
   });
   test('peliasIndexCountryAbbreviation token filters', function (t) {
     var analyzer = settings().analysis.analyzer.peliasIndexCountryAbbreviation;
-    t.deepEqual(analyzer.filter, [
+    t.deepEqual(analyzer.filter, [...mayBeAmpersandReplacer(),
       "lowercase",
       "trim",
       "icu_folding",
@@ -292,13 +307,14 @@ module.exports.tests.peliasIndexCountryAbbreviationOneEdgeGramAnalyzer = functio
     var analyzer = s.analysis.analyzer.peliasIndexCountryAbbreviationOneEdgeGram;
     t.equal(analyzer.type, 'custom', 'custom analyzer');
     t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
-    t.deepEqual(analyzer.char_filter, ["punctuation", "nfkc_normalizer"], 'character filters specified');
+    t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), "punctuation", "nfkc_normalizer"], 'character filters specified');
     t.true(Array.isArray(analyzer.filter), 'filters specified');
     t.end();
   });
   test('peliasIndexCountryAbbreviationOneEdgeGram token filters', function (t) {
     var analyzer = settings().analysis.analyzer.peliasIndexCountryAbbreviationOneEdgeGram;
     t.deepEqual(analyzer.filter, [
+      ...mayBeAmpersandReplacer(),
       "lowercase",
       "trim",
       "icu_folding",
@@ -497,19 +513,6 @@ module.exports.tests.peliasOneEdgeGramFilter = function(test, common) {
   });
 };
 
-// this filter removed leading 0 characters. eg. 0001 -> 1
-module.exports.tests.removeAllZeroNumericPrefixFilter = function(test, common) {
-  test('has removeAllZeroNumericPrefix filter', function(t) {
-    var s = settings();
-    t.equal(typeof s.analysis.filter.removeAllZeroNumericPrefix, 'object', 'there is a removeAllZeroNumericPrefix filter');
-    var filter = s.analysis.filter.removeAllZeroNumericPrefix;
-    t.equal(filter.type, 'pattern_replace');
-    t.equal(filter.pattern, '^(0*)');
-    t.equal(filter.replacement, '');
-    t.end();
-  });
-};
-
 // this filter provides synonyms for street suffixes
 // eg. road=>rd
 module.exports.tests.streetSynonymFilter = function(test, common) {