pelias-schema 8.1.0 → 8.2.0

package/.github/workflows/_integration_tests.yml

@@ -9,7 +9,6 @@ jobs:
           - ubuntu-22.04
         node-version: [18.x, 20.x, 22.x]
         es-version: [7.6.1]
-        jdk-version: [oraclejdk11]
         icuTokenizer: [true, false]
     steps:
     - uses: actions/checkout@v4
@@ -17,10 +16,9 @@ jobs:
       uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
-    - name: Start elasticsearch ${{ matrix.es-version }} (${{ matrix.jdk-version }})
+    - name: Start elasticsearch ${{ matrix.es-version }}
       env:
         ES_VERSION: ${{ matrix.es-version }}
-        JDK_VERSION: ${{ matrix.jdk-version }}
       run: ./scripts/setup_ci.sh
     - name: Run integration tests
       run: |

package/integration/analyzer_peliasIndexOneEdgeGram.js

@@ -52,7 +52,6 @@ module.exports.tests.analyze = function(test, common){
       '4:a', '4:ab', '4:abc', '4:abcd', '4:abcde', '4:abcdef',
       '4:abcdefg', '4:abcdefgh', '4:abcdefghi', '4:abcdefghij'
     ] );
-    assertAnalysis( 'removeAllZeroNumericPrefix', '00001', ['1'] );
 
     assertAnalysis( 'unique', '1 1 1', ['1','1','1'] );
     assertAnalysis( 'notnull', ' / / ', [] );

package/integration/analyzer_peliasPhrase.js

@@ -52,10 +52,6 @@ module.exports.tests.analyze = function(test, common){
     assertAnalysis( 'british_american_english', 'town theatre', ['0:town', '1:theatre', '1:theater'] );
     assertAnalysis( 'british_american_english', 'town theater', ['0:town', '1:theater', '1:theatre'] );
 
-    // remove leading zeros from numeric input
-    assertAnalysis( 'leading_zeros', '01000', ['0:1000'] );
-    assertAnalysis( 'leading_zeros', '09999', ['0:9999'] );
-
     suite.run( t.end );
   });
 };

package/integration/analyzer_peliasQuery.js

@@ -14,6 +14,24 @@ module.exports.tests.analyze = function(test, common){
     var assertAnalysis = common.analyze.bind( null, suite, t, 'peliasQuery' );
     suite.action( function( done ){ setTimeout( done, 500 ); }); // wait for es to bring some shards up
 
+    assertAnalysis('tokenizer', 'foo-bar baz/42', ['foo','bar','baz','42']);
+    assertAnalysis('tokenizer', 'foo-bar  baz/42', ['foo','bar','baz','42']); // tab instead of space
+    assertAnalysis('tokenizer', 'foo---bar baz/42', ['foo','bar','baz','42']);
+    assertAnalysis('tokenizer', 'foo—bar baz/42', ['foobar','baz','42']); // dash is not a hyphen
+    assertAnalysis('tokenizer', 'foo-bar baz//42', ['foo','bar','baz','42']);
+    assertAnalysis('tokenizer', 'foo bar baz 42', ['foo','bar', 'baz', '42']);
+    assertAnalysis('tokenizer', 'foo-bar baz\\42', ['foo', 'bar','baz', '42']);
+    assertAnalysis('thai_digits', '๐๑๒๓๔๕๖๗ ๘๙', ['01234567', '89']); // leading zero remains
+    assertAnalysis('thai_digits', '๑๒๓๔๕๖๗๐ ๘๙', ['12345670', '89']);
+    assertAnalysis('digit_glued_to_word', 'john doe42', ['john', 'doe42']);
+    if (config.schema.icuTokenizer) {
+      assertAnalysis('thai_tonemarks', 'ก่ก้ก๊ก๋ข่ข้ข๊ข๋ค่ค้ค๊ค๋ฆ่ฆ้ฆ๊ฆ๋', ['กก', 'กก', 'ขขขขคคคคฆฆฆฆ']);
+      assertAnalysis('chinese_address', '北京市朝阳区东三环中路1号国际大厦A座1001室', ['北京市', '朝阳', '区', '东', '三', '环', '中路', '1', '号', '国际', '大厦', 'a', '座', '1001', '室']);  
+    } else {
+      assertAnalysis('thai_tonemarks', 'ก่ก้ก๊ก๋ข่ข้ข๊ข๋ค่ค้ค๊ค๋ฆ่ฆ้ฆ๊ฆ๋', ['กกกกขขขขคคคคฆฆฆฆ']);
+      assertAnalysis('chinese_address', '北京市朝阳区东三环中路1号国际大厦A座1001室', ['北京市朝阳区东三环中路1号国际大厦a座1001室']);  
+    }
+
     assertAnalysis('asciifolding', 'é', ['e']);
     assertAnalysis('asciifolding', 'ß', ['ss']);
     assertAnalysis('asciifolding', 'æ', ['ae']);
@@ -23,7 +41,6 @@ module.exports.tests.analyze = function(test, common){
     assertAnalysis('trim', ' f ', ['f']);
     assertAnalysis('remove_ordinals', '26t', ['26']);
     assertAnalysis('remove_ordinals', '26th', ['26']);
-    assertAnalysis('removeAllZeroNumericPrefix', '00001', ['1']);
     assertAnalysis('unique', '1 1 1', ['1','1','1']);
     assertAnalysis('notnull', ' / / ', []);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pelias-schema",
-  "version": "8.1.0",
+  "version": "8.2.0",
   "author": "pelias",
   "description": "Elasticsearch schema files and tooling for Pelias",
   "homepage": "https://github.com/pelias/schema",

package/scripts/setup_ci.sh

@@ -4,9 +4,6 @@ set -e
 # create elasticsearch directory
 mkdir /tmp/elasticsearch
 
-# allow switching the JDK version
-curl -s https://raw.githubusercontent.com/michaelklishin/jdk_switcher/master/jdk_switcher.sh | bash -s use "${JDK_VERSION}"
-
 # download and install elasticsearch with ICU plugin
 FILENAME="elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz"
 STRIP_COMPONENTS=1

package/settings.js

@@ -57,7 +57,6 @@ function generate(){
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "peliasOneEdgeGramFilter",
             "unique_only_same_position",
             "notnull",
@@ -73,7 +72,6 @@ function generate(){
             "trim",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "unique_only_same_position",
             "notnull"
           ]
@@ -92,7 +90,6 @@ function generate(){
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "unique_only_same_position",
             "notnull",
             "flatten_graph"
@@ -231,11 +228,6 @@ function generate(){
           "min_gram" : 1,
           "max_gram" : 24
         },
-        "removeAllZeroNumericPrefix" :{
-          "type" : "pattern_replace",
-          "pattern" : "^(0*)",
-          "replacement" : ""
-        },
         "remove_ordinals" : {
           "type" : "pattern_replace",
           "pattern": "(?i)((^| )((1)st?|(2)nd?|(3)rd?|([4-9])th?)|(([0-9]*)(1[0-9])th?)|(([0-9]*[02-9])((1)st?|(2)nd?|(3)rd?|([04-9])th?))($| ))",

package/test/fixtures/expected-icu-tokenizer.json

@@ -58,7 +58,6 @@
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "peliasOneEdgeGramFilter",
             "unique_only_same_position",
             "notnull",
@@ -79,7 +78,6 @@
             "trim",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "unique_only_same_position",
             "notnull"
           ]
@@ -103,7 +101,6 @@
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "unique_only_same_position",
             "notnull",
             "flatten_graph"
@@ -270,11 +267,6 @@
           "min_gram": 1,
           "max_gram": 24
         },
-        "removeAllZeroNumericPrefix": {
-          "type": "pattern_replace",
-          "pattern": "^(0*)",
-          "replacement": ""
-        },
         "remove_ordinals": {
           "type": "pattern_replace",
           "pattern": "(?i)((^| )((1)st?|(2)nd?|(3)rd?|([4-9])th?)|(([0-9]*)(1[0-9])th?)|(([0-9]*[02-9])((1)st?|(2)nd?|(3)rd?|([04-9])th?))($| ))",

package/test/fixtures/expected.json

@@ -55,7 +55,6 @@
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "peliasOneEdgeGramFilter",
             "unique_only_same_position",
             "notnull",
@@ -74,7 +73,6 @@
             "trim",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "unique_only_same_position",
             "notnull"
           ]
@@ -96,7 +94,6 @@
             "name_synonyms_multiplexer",
             "icu_folding",
             "remove_ordinals",
-            "removeAllZeroNumericPrefix",
             "unique_only_same_position",
             "notnull",
             "flatten_graph"
@@ -252,11 +249,6 @@
           "min_gram": 1,
           "max_gram": 24
         },
-        "removeAllZeroNumericPrefix": {
-          "type": "pattern_replace",
-          "pattern": "^(0*)",
-          "replacement": ""
-        },
         "remove_ordinals": {
           "type": "pattern_replace",
           "pattern": "(?i)((^| )((1)st?|(2)nd?|(3)rd?|([4-9])th?)|(([0-9]*)(1[0-9])th?)|(([0-9]*[02-9])((1)st?|(2)nd?|(3)rd?|([04-9])th?))($| ))",

package/test/settings.js

@@ -116,7 +116,6 @@ module.exports.tests.peliasIndexOneEdgeGramAnalyzer = function(test, common) {
       "name_synonyms_multiplexer",
       "icu_folding",
       "remove_ordinals",
-      "removeAllZeroNumericPrefix",
       "peliasOneEdgeGramFilter",
       "unique_only_same_position",
       "notnull",
@@ -145,7 +144,6 @@ module.exports.tests.peliasQueryAnalyzer = function (test, common) {
       'trim',
       'icu_folding',
       'remove_ordinals',
-      'removeAllZeroNumericPrefix',
       'unique_only_same_position',
       'notnull'
     ]);
@@ -177,7 +175,6 @@ module.exports.tests.peliasPhraseAnalyzer = function(test, common) {
       "name_synonyms_multiplexer",
       "icu_folding",
       "remove_ordinals",
-      "removeAllZeroNumericPrefix",
       "unique_only_same_position",
       "notnull",
       "flatten_graph"
@@ -516,19 +513,6 @@ module.exports.tests.peliasOneEdgeGramFilter = function(test, common) {
   });
 };
 
-// this filter removed leading 0 characters. eg. 0001 -> 1
-module.exports.tests.removeAllZeroNumericPrefixFilter = function(test, common) {
-  test('has removeAllZeroNumericPrefix filter', function(t) {
-    var s = settings();
-    t.equal(typeof s.analysis.filter.removeAllZeroNumericPrefix, 'object', 'there is a removeAllZeroNumericPrefix filter');
-    var filter = s.analysis.filter.removeAllZeroNumericPrefix;
-    t.equal(filter.type, 'pattern_replace');
-    t.equal(filter.pattern, '^(0*)');
-    t.equal(filter.replacement, '');
-    t.end();
-  });
-};
-
 // this filter provides synonyms for street suffixes
 // eg. road=>rd
 module.exports.tests.streetSynonymFilter = function(test, common) {