pelias-schema 9.1.0 → 10.1.0

package/.github/workflows/_test.yml

@@ -0,0 +1,21 @@
+name: Unit Tests
+on: workflow_call
+jobs:
+  unit-tests:
+    runs-on: '${{ matrix.os }}'
+    strategy:
+      matrix:
+        os:
+          - ubuntu-24.04
+        node-version: [ 22.x, 24.x, 26.x ]
+    steps:
+      - uses: actions/checkout@v6
+      - name: 'Install node.js ${{ matrix.node-version }}'
+        uses: actions/setup-node@v6
+        with:
+          node-version: '${{ matrix.node-version }}'
+      - name: Run unit tests
+        run: |
+          [[ -f ./bin/ci-setup ]] && ./bin/ci-setup
+          npm install
+          npm run ci

package/.github/workflows/pull_request.yml

@@ -4,8 +4,4 @@ jobs:
   unit-tests:
     # only run this job for forks
     if: github.event.pull_request.head.repo.full_name != github.repository
-    uses: ./.github/workflows/_unit_tests.yml
-  integration-tests:
-    # only run this job for forks
-    if: github.event.pull_request.head.repo.full_name != github.repository
-    uses: ./.github/workflows/_integration_tests.yml
+    uses: ./.github/workflows/_test.yml

package/.github/workflows/push.yml

@@ -1,37 +1,36 @@
-name: 'Continuous Integration'
-on:
-  push:
-    branches-ignore:
-      - 'v[0-9]+.[0-9]+.[0-9]+'
+name: Continuous Integration
+on: push
 jobs:
   unit-tests:
-    uses: ./.github/workflows/_unit_tests.yml
-  integration-tests:
-    uses: ./.github/workflows/_integration_tests.yml
+    uses: ./.github/workflows/_test.yml
   npm-publish:
-    needs: [unit-tests, integration-tests]
-    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-    runs-on: ubuntu-22.04
+    needs: unit-tests
+    if: github.ref == 'refs/heads/master' && needs.unit-tests.result == 'success'
+    runs-on: ubuntu-24.04
+    permissions:
+      id-token: write
+      contents: write
     steps:
-    - uses: actions/checkout@v4
-    - name: Install Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: 20.x
-    - name: Run semantic-release
-      env:
-        GH_TOKEN: ${{ secrets.GH_SEMANTIC_RELEASE_TOKEN }}
-        NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-      run: |
-        curl "https://raw.githubusercontent.com/pelias/ci-tools/master/semantic-release.sh" | bash -
+      - uses: actions/checkout@v6
+      - name: Install Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 22.x
+      - name: Run semantic-release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: >
+          if [[ "${{ github.repository_owner }}" == "pelias" ]]; then
+            curl "https://raw.githubusercontent.com/pelias/ci-tools/master/semantic-release.sh" | bash -
+          fi
   build-docker-images:
     # run this job if the unit tests passed and the npm-publish job was a success or was skipped
     # note: github actions won't run a job if you don't call one of the status check functions, so `always()` is called since it evalutes to `true`
     if: ${{ always() && needs.unit-tests.result == 'success' && (needs.npm-publish.result == 'success' || needs.npm-publish.result == 'skipped') }}
-    needs: [unit-tests, integration-tests, npm-publish]
-    runs-on: ubuntu-22.04
+    needs: [unit-tests, npm-publish]
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Build Docker images
         env:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}

package/.jshintrc

@@ -2,7 +2,7 @@
   "node": true,
   "curly": true,
   "eqeqeq": true,
-  "esversion": 6,
+  "esversion": 11,
   "freeze": true,
   "immed": true,
   "indent": 2,
@@ -13,8 +13,10 @@
   "nonbsp": true,
   "nonew": true,
   "plusplus": false,
+  "quotmark": "single",
   "undef": true,
   "unused": false,
   "maxparams": 4,
-  "maxdepth": 4
+  "maxdepth": 4,
+  "maxlen": 140
 }

package/bin/ci

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# run the unit and integration tests against both icuTokenizer settings
+set -euo pipefail
+
+BASE_CONFIG="${PELIAS_CONFIG:-${HOME}/pelias.json}"
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "${WORKDIR}"' EXIT
+
+# write a copy of the config with the icuTokenizer setting overridden
+write_config() {
+  local base='{}'
+  if [[ -f "${BASE_CONFIG}" ]]; then base="$(cat "${BASE_CONFIG}")"; fi
+  jq -n --argjson base "${base}" --argjson icu "$1" \
+    '$base * { schema: { icuTokenizer: $icu } }' > "${WORKDIR}/pelias.json"
+}
+
+for ICU_TOKENIZER in false true; do
+  echo "--- icuTokenizer: ${ICU_TOKENIZER} ---"
+  write_config "${ICU_TOKENIZER}"
+  export PELIAS_CONFIG="${WORKDIR}/pelias.json"
+
+  npm run test
+
+  # the index is recreated per variant to assert the schema is valid for both
+  node scripts/drop_index --force-yes &> /dev/null || true
+  npm run create_index
+  npm run integration
+done

package/{scripts/setup_ci.sh → bin/ci-setup}

@@ -1,8 +1,11 @@
 #!/bin/bash
-set -e
+set -eo pipefail
+
+# the integration tests require a running elasticsearch server
+ES_VERSION="${ES_VERSION:-7.6.1}"
 
 # create elasticsearch directory
-mkdir /tmp/elasticsearch
+mkdir -p /tmp/elasticsearch
 
 # download and install elasticsearch with ICU plugin
 FILENAME="elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz"
@@ -25,7 +28,7 @@ wget -O - "https://artifacts.elastic.co/downloads/elasticsearch/${FILENAME}" \
 
 # wait for server to boot up
 # logs show that on travis-ci it can take ~17s to boot an ES6 server
-source "${BASH_SOURCE%/*}/elastic_wait.sh"
+source "${BASH_SOURCE%/*}/../scripts/elastic_wait.sh"
 (elastic_wait)
 
 # set the correct esclient.apiVersion in pelias.json

package/configValidation.js

@@ -3,11 +3,15 @@ const Joi = require('@hapi/joi');
 // Schema Configuration
 // schema.indexName: populated by defaults if not overridden
 // schema.icuTokenizer: boolean, optional, defaults to false
+// schema.unstoredFields: array of field paths to index but omit from _source, optional
+// schema.excludedFields: array of field paths to remove from the mapping, optional
 // esclient: object, validation performed by elasticsearch module
 const schema = Joi.object().required().keys({
   schema: Joi.object().required().keys({
     indexName: Joi.string().required(),
-    icuTokenizer: Joi.boolean().optional()
+    icuTokenizer: Joi.boolean().optional(),
+    unstoredFields: Joi.array().items(Joi.string()).optional(),
+    excludedFields: Joi.array().items(Joi.string()).optional()
   }),
   esclient: Joi.object().required()
 }).unknown(true);

package/mappings/configureFields.js

@@ -0,0 +1,28 @@
+const _ = require('lodash');
+
+// convert a dotted field path such as 'parent.county_a' into its location in a mapping
+const mappingPath = (field) => _.flatMap(field.split('.'), (part) => ['properties', part]);
+
+/**
+ * apply field configuration from `schema` section of pelias config to a mapping
+ *
+ * unstoredFields: fields that are indexed but omitted from _source
+ * excludedFields: fields removed from the mapping entirely
+ */
+function configureFields(mapping, schemaConfig) {
+  const unstored = _.get(schemaConfig, 'unstoredFields', []);
+  const excluded = _.get(schemaConfig, 'excludedFields', []);
+
+  const configured = _.cloneDeep(mapping);
+
+  if (!_.isEmpty(unstored)) {
+    const excludes = _.get(configured, '_source.excludes', []);
+    _.set(configured, '_source.excludes', _.union(excludes, unstored));
+  }
+
+  excluded.forEach((field) => _.unset(configured, mappingPath(field)));
+
+  return configured;
+}
+
+module.exports = configureFields;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pelias-schema",
-  "version": "9.1.0",
+  "version": "10.1.0",
   "author": "pelias",
   "description": "Elasticsearch schema files and tooling for Pelias",
   "homepage": "https://github.com/pelias/schema",
@@ -8,6 +8,7 @@
   "main": "schema.js",
   "scripts": {
     "test": "./bin/test",
+    "ci": "./bin/ci",
     "integration": "./bin/integration",
     "create_index": "./bin/create_index",
     "drop_index": "node scripts/drop_index",
@@ -25,7 +26,7 @@
     "url": "https://github.com/pelias/schema/issues"
   },
   "engines": {
-    "node": ">=10.0.0"
+    "node": ">=22.0.0"
   },
   "dependencies": {
     "@hapi/joi": "^16.1.8",
@@ -33,8 +34,8 @@
     "elasticsearch": "^16.0.0",
     "glob": "^7.1.6",
     "lodash": "^4.17.15",
-    "pelias-config": "^6.0.0",
-    "pelias-logger": "^1.3.0",
+    "pelias-config": "^7.0.0",
+    "pelias-logger": "^2.0.0",
     "semver": "^7.1.1"
   },
   "devDependencies": {

package/schema.js

@@ -1,6 +1,9 @@
+const config = require('pelias-config').generate();
+const configureFields = require('./mappings/configureFields');
+
 const schema = {
   settings: require('./settings')(),
-  mappings: require('./mappings/document'),
+  mappings: configureFields(require('./mappings/document'), config.schema),
 };
 
 module.exports = schema;

package/test/configValidation.js

@@ -84,6 +84,43 @@ module.exports.tests.interface = function(test, common) {
 
   });
 
+  test('config with array of strings for unstoredFields and excludedFields should not throw error', function(t) {
+    var config = {
+      schema: {
+        indexName: 'example_index',
+        unstoredFields: ['addendum', 'parent.county_a'],
+        excludedFields: ['popularity']
+      },
+      esclient: {}
+    };
+
+    t.doesNotThrow(function() {
+      configValidation.validate(config);
+    }, 'no error should have been thrown');
+
+    t.end();
+  });
+
+  test('config with non-array unstoredFields or excludedFields should throw error', function(t) {
+    ['unstoredFields', 'excludedFields'].forEach((key) => {
+      [null, 17, 'addendum', {}, true, [17]].forEach((value) => {
+        var config = {
+          schema: {
+            indexName: 'example_index',
+            [key]: value
+          },
+          esclient: {}
+        };
+
+        t.throws(function() {
+          configValidation.validate(config);
+        }, new RegExp(`"schema.${key}(\\[0\\])?" must be`), `${key} must be an array of strings`);
+      });
+    });
+
+    t.end();
+  });
+
 };
 
 module.exports.all = function (tape, common) {

package/test/configureFields.js

@@ -0,0 +1,49 @@
+const configureFields = require('../mappings/configureFields');
+const document = require('../mappings/document');
+
+module.exports.tests = {};
+
+module.exports.tests.unstored = function(test, common) {
+  test('no config leaves mapping unchanged', function(t) {
+    t.deepEqual(configureFields(document, undefined), document);
+    t.deepEqual(configureFields(document, {}), document);
+    t.end();
+  });
+
+  test('unstored fields are appended to _source excludes', function(t) {
+    const mapping = configureFields(document, { unstoredFields: ['addendum', 'parent.*_a', 'shape'] });
+    t.deepEqual(mapping._source.excludes, ['shape', 'phrase', 'addendum', 'parent.*_a']);
+    t.deepEqual(mapping.properties, document.properties, 'properties unchanged');
+    t.end();
+  });
+};
+
+module.exports.tests.excluded = function(test, common) {
+  test('excluded fields are removed from mapping', function(t) {
+    const mapping = configureFields(document, { excludedFields: ['popularity', 'parent.county_a', 'not_a_field'] });
+    t.false(mapping.properties.popularity, 'top level field removed');
+    t.false(mapping.properties.parent.properties.county_a, 'nested field removed');
+    t.true(mapping.properties.parent.properties.county, 'sibling field kept');
+    t.true(mapping.properties.population, 'other fields kept');
+    t.end();
+  });
+
+  test('original mapping is not modified', function(t) {
+    configureFields(document, { unstoredFields: ['addendum'], excludedFields: ['popularity', 'parent.county_a'] });
+    t.deepEqual(document._source.excludes, ['shape', 'phrase']);
+    t.true(document.properties.popularity);
+    t.true(document.properties.parent.properties.county_a);
+    t.end();
+  });
+};
+
+module.exports.all = function (tape, common) {
+
+  function test(name, testFunction) {
+    return tape('configureFields: ' + name, testFunction);
+  }
+
+  for( var testCase in module.exports.tests ){
+    module.exports.tests[testCase](test, common);
+  }
+};

package/test/run.js

@@ -11,6 +11,7 @@ var common = {
 var tests = [
   require('./compile.js'),
   require('./document.js'),
+  require('./configureFields.js'),
   require('./partial-centroid.js'),
   require('./partial-admin.js'),
   require('./partial-keyword.js'),

package/.github/workflows/_integration_tests.yml

@@ -1,32 +0,0 @@
-name: Integration Tests
-on: workflow_call
-jobs:
-  integration-tests:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os:
-          - ubuntu-22.04
-        node-version: [ 20.x, 22.x, 24.x ]
-        es-version: [7.6.1]
-        icuTokenizer: [true, false]
-    steps:
-    - uses: actions/checkout@v4
-    - name: Install node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v4
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Start elasticsearch ${{ matrix.es-version }}
-      env:
-        ES_VERSION: ${{ matrix.es-version }}
-      run: ./scripts/setup_ci.sh
-    - name: Run integration tests
-      run: |
-        if [ "${{ matrix.icuTokenizer }}" = "true" ]; then
-          jq -n '{ schema: { icuTokenizer: true } }' > $(pwd)/config-icu.json
-          export PELIAS_CONFIG=$(pwd)/config-icu.json
-        fi
-        npm install
-        curl http://127.0.0.1:9200/
-        ./bin/create_index
-        npm run integration

package/.github/workflows/_unit_tests.yml

@@ -1,25 +0,0 @@
-name: Unit Tests
-on: workflow_call
-jobs:
-  unit-tests:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os:
-          - ubuntu-22.04
-        node-version: [ 20.x, 22.x, 24.x ]
-        icuTokenizer: [true, false]
-    steps:
-    - uses: actions/checkout@v4
-    - name: Install node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v4
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: Run unit tests
-      run: |
-        npm install
-        if [ "${{ matrix.icuTokenizer }}" = "true" ]; then
-          jq -n '{ schema: { icuTokenizer: true } }' > $(pwd)/config-icu.json
-          export PELIAS_CONFIG=$(pwd)/config-icu.json
-        fi
-        npm run test