pelias-schema 7.0.0 → 7.2.0

package/.github/workflows/_integration_tests.yml

@@ -7,13 +7,13 @@ jobs:
       matrix:
         os:
           - ${{ vars.UBUNTU_VERSION }}
-        node-version: [16.x, 18.x]
+        node-version: [18.x, 20.x, 22.x]
         es-version: [7.6.1]
         jdk-version: [oraclejdk11]
     steps:
     - uses: actions/checkout@v2
     - name: Install node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2-beta
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
     - name: Start elasticsearch ${{ matrix.es-version }} (${{ matrix.jdk-version }})

package/.github/workflows/_unit_tests.yml

@@ -7,11 +7,11 @@ jobs:
       matrix:
         os:
           - ${{ vars.UBUNTU_VERSION }}
-        node-version: [16.x, 18.x]
+        node-version: [18.x, 20.x, 22.x]
     steps:
     - uses: actions/checkout@v2
     - name: Install node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v2-beta
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
     - name: Run unit tests

package/.github/workflows/push.yml

@@ -15,7 +15,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Install Node.js
-      uses: actions/setup-node@v2-beta
+      uses: actions/setup-node@v4
       with:
         node-version: 20.x
     - name: Run semantic-release

package/integration/analyzer_peliasPhrase.js

@@ -47,7 +47,8 @@ module.exports.tests.analyze = function(test, common){
     // remove punctuation (handled by the char_filter)
     assertAnalysis( 'punctuation', punctuation.all.join(''), ['0:&', '0:and', '0:und'] );
     assertAnalysis( 'punctuation', 'Hawai‘i', ['hawaii'] );
-
+    assertAnalysis( 'punctuation - « in between', '«res»pub«lika»', ['respublika'] );
+ 
     assertAnalysis( 'british_american_english', 'town theatre', ['0:town', '1:theatre', '1:theater'] );
     assertAnalysis( 'british_american_english', 'town theater', ['0:town', '1:theater', '1:theatre'] );
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pelias-schema",
-  "version": "7.0.0",
+  "version": "7.2.0",
   "author": "pelias",
   "description": "Elasticsearch schema files and tooling for Pelias",
   "homepage": "https://github.com/pelias/schema",

package/punctuation.js

@@ -1,23 +1,18 @@
 // These characters will be removed from ngrams/shingles
 // @see: org/apache/lucene/analysis/cn/smart/stopwords.txt
 
-module.exports.all = [
-  ".","`","‘","-","_","=","?","'","|","\"","(",")","{","}","[","]","<",">","*",
-  "#","&","^","$","@","!","~",":",";","+","《","》","—","－","，","。",
-  "、", "：","；","！","·","？","„","“","”","）","（","【","】","［","］","●"
+const all = [
+  ".","`","‘","’","‛","-","_","=","?","'","|","\"","(",")","{","}","[","]","<",">","*",
+  "#","&","^","$","@","!","~",":",";","+","《","》","—","－","，","。","‹","›","⹂","〝","〞",
+  "、", "：","；","！","·","？","„","“","”","‟","）","（","【","】","［","］","●","«","»"
 ];
 
-module.exports.allowed = [
+const allowed = [
   "-", // allow hypens
   "&"  // allow ampersands
 ];
 
-module.exports.blacklist = module.exports.all.slice();
-
 // remove alowed chars from blacklist
-module.exports.allowed.forEach(function(item){
-  var index = module.exports.blacklist.indexOf(item);
-  if( index > -1 ){
-    module.exports.blacklist.splice(index, 1);
-  }
-});
+const blacklist = all.filter(s => !allowed.includes(s));
+
+module.exports = { all, allowed, blacklist };

package/scripts/create_index.js

@@ -38,5 +38,5 @@ client.indices.create(req, (err, res) => {
     process.exit(1);
   }
   console.log('[put mapping]', '\t', indexName, res, '\n');
-  process.exit(!!err);
+  process.exit(0);
 });

package/test/fixtures/expected.json

@@ -2277,6 +2277,8 @@
             ".=>",
             "`=>",
             "‘=>",
+            "’=>",
+            "‛=>",
             "_=>",
             "==>",
             "?=>",
@@ -2307,6 +2309,11 @@
             "－=>",
             "，=>",
             "。=>",
+            "‹=>",
+            "›=>",
+            "⹂=>",
+            "〝=>",
+            "〞=>",
             "、=>",
             "：=>",
             "；=>",
@@ -2316,13 +2323,16 @@
             "„=>",
             "“=>",
             "”=>",
+            "‟=>",
             "）=>",
             "（=>",
             "【=>",
             "】=>",
             "［=>",
             "］=>",
-            "●=>"
+            "●=>",
+            "«=>",
+            "»=>"
           ]
         },
         "alphanumeric": {
@@ -3023,4 +3033,4 @@
     },
     "dynamic": "strict"
   }
-}
+}

package/test/settings.js

@@ -591,7 +591,7 @@ module.exports.tests.punctuationCharFilter = function(test, common) {
     var char_filter = s.analysis.char_filter.punctuation;
     t.equal(char_filter.type, 'mapping');
     t.true(Array.isArray(char_filter.mappings));
-    t.equal(char_filter.mappings.length, 49);
+    t.equal(char_filter.mappings.length, 59);
     t.end();
   });
 };