pelias-schema 8.0.0 → 8.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. package/.github/workflows/_integration_tests.yml +5 -0
  2. package/.github/workflows/_unit_tests.yml +6 -1
  3. package/configValidation.js +2 -0
  4. package/integration/analyzer_peliasIndexOneEdgeGram.js +12 -2
  5. package/integration/analyzer_peliasQuery.js +30 -2
  6. package/integration/analyzer_peliasStreet.js +15 -0
  7. package/package.json +1 -1
  8. package/settings-icu.js +49 -0
  9. package/settings.js +8 -2
  10. package/test/compile.js +39 -4
  11. package/test/fixtures/config-icu-tokenizer.json +15 -0
  12. package/test/fixtures/expected-icu-tokenizer.json +3059 -0
  13. package/test/settings.js +31 -12
package/test/settings.js CHANGED
@@ -1,6 +1,7 @@
1
- var path = require('path'),
1
+ const path = require('path'),
2
2
  settings = require('../settings'),
3
- fs = require('fs');
3
+ fs = require('fs'),
4
+ config = require('pelias-config').generate();
4
5
 
5
6
  module.exports.tests = {};
6
7
 
@@ -48,6 +49,20 @@ module.exports.tests.analysis = function(test, common) {
48
49
  });
49
50
  };
50
51
 
52
+ function mayBeAmpersandMapper() {
53
+ if (config.schema.icuTokenizer) {
54
+ return ['ampersand_mapper'];
55
+ }
56
+ return [];
57
+ }
58
+
59
+ function mayBeAmpersandReplacer() {
60
+ if (config.schema.icuTokenizer) {
61
+ return ['ampersand_replacer'];
62
+ }
63
+ return [];
64
+ }
65
+
51
66
  // -- analyzers --
52
67
 
53
68
  module.exports.tests.peliasAdminAnalyzer = function(test, common) {
@@ -57,13 +72,13 @@ module.exports.tests.peliasAdminAnalyzer = function(test, common) {
57
72
  var analyzer = s.analysis.analyzer.peliasAdmin;
58
73
  t.equal(analyzer.type, 'custom', 'custom analyzer');
59
74
  t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
60
- t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified');
75
+ t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), 'punctuation', 'nfkc_normalizer'], 'character filters specified');
61
76
  t.true(Array.isArray(analyzer.filter), 'filters specified');
62
77
  t.end();
63
78
  });
64
79
  test('peliasAdmin token filters', function (t) {
65
80
  var analyzer = settings().analysis.analyzer.peliasAdmin;
66
- t.deepEqual(analyzer.filter, [
81
+ t.deepEqual(analyzer.filter, [...mayBeAmpersandReplacer(),
67
82
  "lowercase",
68
83
  "trim",
69
84
  "synonyms/custom_admin/multiword",
@@ -85,13 +100,14 @@ module.exports.tests.peliasIndexOneEdgeGramAnalyzer = function(test, common) {
85
100
  var analyzer = s.analysis.analyzer.peliasIndexOneEdgeGram;
86
101
  t.equal(analyzer.type, 'custom', 'custom analyzer');
87
102
  t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
88
- t.deepEqual(analyzer.char_filter, ["punctuation","nfkc_normalizer"], 'character filters specified');
103
+ t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), "punctuation","nfkc_normalizer"], 'character filters specified');
89
104
  t.true(Array.isArray(analyzer.filter), 'filters specified');
90
105
  t.end();
91
106
  });
92
107
  test('peliasIndexOneEdgeGram token filters', function(t) {
93
108
  var analyzer = settings().analysis.analyzer.peliasIndexOneEdgeGram;
94
109
  t.deepEqual( analyzer.filter, [
110
+ ...mayBeAmpersandReplacer(),
95
111
  "lowercase",
96
112
  "trim",
97
113
  "synonyms/custom_name/multiword",
@@ -117,13 +133,14 @@ module.exports.tests.peliasQueryAnalyzer = function (test, common) {
117
133
  var analyzer = s.analysis.analyzer.peliasQuery;
118
134
  t.equal(analyzer.type, 'custom', 'custom analyzer');
119
135
  t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
120
- t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified');
136
+ t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), 'punctuation', 'nfkc_normalizer'], 'character filters specified');
121
137
  t.true(Array.isArray(analyzer.filter), 'filters specified');
122
138
  t.end();
123
139
  });
124
140
  test('peliasQuery token filters', function (t) {
125
141
  var analyzer = settings().analysis.analyzer.peliasQuery;
126
142
  t.deepEqual(analyzer.filter, [
143
+ ...mayBeAmpersandReplacer(),
127
144
  'lowercase',
128
145
  'trim',
129
146
  'icu_folding',
@@ -143,13 +160,14 @@ module.exports.tests.peliasPhraseAnalyzer = function(test, common) {
143
160
  var analyzer = s.analysis.analyzer.peliasPhrase;
144
161
  t.equal(analyzer.type, 'custom', 'custom analyzer');
145
162
  t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
146
- t.deepEqual(analyzer.char_filter, ["punctuation","nfkc_normalizer"], 'character filters specified');
163
+ t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), "punctuation", "nfkc_normalizer"], 'character filters specified');
147
164
  t.true(Array.isArray(analyzer.filter), 'filters specified');
148
165
  t.end();
149
166
  });
150
167
  test('peliasPhrase token filters', function(t) {
151
168
  var analyzer = settings().analysis.analyzer.peliasPhrase;
152
169
  t.deepEqual( analyzer.filter, [
170
+ ...mayBeAmpersandReplacer(),
153
171
  "lowercase",
154
172
  "trim",
155
173
  "remove_duplicate_spaces",
@@ -236,13 +254,13 @@ module.exports.tests.peliasStreetAnalyzer = function(test, common) {
236
254
  var analyzer = s.analysis.analyzer.peliasStreet;
237
255
  t.equal(analyzer.type, 'custom', 'custom analyzer');
238
256
  t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
239
- t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified');
257
+ t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), 'punctuation', 'nfkc_normalizer'], 'character filters specified');
240
258
  t.true(Array.isArray(analyzer.filter), 'filters specified');
241
259
  t.end();
242
260
  });
243
261
  test('peliasStreet token filters', function(t) {
244
262
  var analyzer = settings().analysis.analyzer.peliasStreet;
245
- t.deepEqual( analyzer.filter, [
263
+ t.deepEqual( analyzer.filter, [...mayBeAmpersandReplacer(),
246
264
  "lowercase",
247
265
  "trim",
248
266
  "remove_duplicate_spaces",
@@ -266,13 +284,13 @@ module.exports.tests.peliasIndexCountryAbbreviation = function (test, common) {
266
284
  var analyzer = s.analysis.analyzer.peliasIndexCountryAbbreviation;
267
285
  t.equal(analyzer.type, 'custom', 'custom analyzer');
268
286
  t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
269
- t.deepEqual(analyzer.char_filter, ['punctuation', 'nfkc_normalizer'], 'character filters specified');
287
+ t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), 'punctuation', 'nfkc_normalizer'], 'character filters specified');
270
288
  t.true(Array.isArray(analyzer.filter), 'filters specified');
271
289
  t.end();
272
290
  });
273
291
  test('peliasIndexCountryAbbreviation token filters', function (t) {
274
292
  var analyzer = settings().analysis.analyzer.peliasIndexCountryAbbreviation;
275
- t.deepEqual(analyzer.filter, [
293
+ t.deepEqual(analyzer.filter, [...mayBeAmpersandReplacer(),
276
294
  "lowercase",
277
295
  "trim",
278
296
  "icu_folding",
@@ -292,13 +310,14 @@ module.exports.tests.peliasIndexCountryAbbreviationOneEdgeGramAnalyzer = functio
292
310
  var analyzer = s.analysis.analyzer.peliasIndexCountryAbbreviationOneEdgeGram;
293
311
  t.equal(analyzer.type, 'custom', 'custom analyzer');
294
312
  t.equal(typeof analyzer.tokenizer, 'string', 'tokenizer specified');
295
- t.deepEqual(analyzer.char_filter, ["punctuation", "nfkc_normalizer"], 'character filters specified');
313
+ t.deepEqual(analyzer.char_filter, [...mayBeAmpersandMapper(), "punctuation", "nfkc_normalizer"], 'character filters specified');
296
314
  t.true(Array.isArray(analyzer.filter), 'filters specified');
297
315
  t.end();
298
316
  });
299
317
  test('peliasIndexCountryAbbreviationOneEdgeGram token filters', function (t) {
300
318
  var analyzer = settings().analysis.analyzer.peliasIndexCountryAbbreviationOneEdgeGram;
301
319
  t.deepEqual(analyzer.filter, [
320
+ ...mayBeAmpersandReplacer(),
302
321
  "lowercase",
303
322
  "trim",
304
323
  "icu_folding",