pelias-openstreetmap 8.3.0 → 8.3.2

package/.github/workflows/_test.yml

@@ -6,12 +6,12 @@ jobs:
     strategy:
       matrix:
         os:
-          - ubuntu-22.04
+          - ubuntu-24.04
         node-version: [ 20.x, 22.x, 24.x ]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: 'Install node.js ${{ matrix.node-version }}'
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: '${{ matrix.node-version }}'
       - name: Run unit tests

package/.github/workflows/push.yml

@@ -6,19 +6,21 @@ jobs:
   npm-publish:
     needs: unit-tests
     if: github.ref == 'refs/heads/master' && needs.unit-tests.result == 'success'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
+    permissions:
+      id-token: write
+      contents: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Install Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
-          node-version: 20.x
+          node-version: 22.x
       - name: Run semantic-release
         env:
-          GH_TOKEN: ${{ secrets.GH_SEMANTIC_RELEASE_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: >
-          if [[ -n "$GH_TOKEN" && -n "$NPM_TOKEN" ]]; then
+          if [[ "${{ github.repository_owner }}" == "pelias" ]]; then
             curl "https://raw.githubusercontent.com/pelias/ci-tools/master/semantic-release.sh" | bash -
           fi
   build-docker-images:
@@ -26,9 +28,9 @@ jobs:
     # note: github actions won't run a job if you don't call one of the status check functions, so `always()` is called since it evalutes to `true`
     if: ${{ always() && needs.unit-tests.result == 'success' && (needs.npm-publish.result == 'success' || needs.npm-publish.result == 'skipped') }}
     needs: [unit-tests, npm-publish]
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Build Docker images
         env:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}

package/README.md

@@ -94,6 +94,24 @@ The OSM importer will look for a file with a name matching this value in the con
 
 If downloading from a remote URL, the filename must match the value in `sourceURL`.
 
+#### `imports.openstreetmap.layers`
+
+This is an object you can use to define your own layers based on OSM tags. For example, if you wanted a layer `coffee` for just coffee shops you could set:
+
+```
+{
+  "imports": {
+    "openstreetmap": {
+      "layers": {
+        "coffee": {
+          "tags": [ "amenity~cafe+name" ]
+        }
+      }
+    }
+  }
+}
+```
+
 #### `imports.openstreetmap.leveldbpath`
 
 This is the directory where temporary files will be stored in order to
@@ -106,6 +124,8 @@ Defaults to `tmp`.
 
 By default, the OSM importer imports both venue records and addresses. If set to false, only address records will be imported.
 
+You can also disable venue import by setting `imports.openstreetmap.layers.venue` to an empty value.
+
 #### `imports.openstreetmap.removeDisusedVenues`
 
 If set to boolean `true`, `venue`s with popularity below `0` (as determined by OSM tags) will be

package/config/features.js

@@ -2,52 +2,60 @@
 /**
  default list of tags to extract from the pbf file when running
  imports. @see: https://github.com/pelias/pbf2json for more info.
-**/
 
-// default tags imported
-const tags = [
-  'addr:housenumber+addr:street',
-  'addr:housenumber+addr:place'  // @ref https://github.com/pelias/pelias/issues/787#issuecomment-477137803
-];
+ addressTags: extracted so address_extractor can assign the address layer downstream.
+              these are not a layer themselves — address classification is structural,
+              not tag-based.
 
-// tags corresponding to venues
-const venue_tags = [
-  'amenity+name',
-  'building+name',
-  'shop+name',
-  'office+name',
-  'public_transport+name',
-  'cuisine+name',
-  'railway~station+name',
-  'railway~tram_stop+name',
-  'railway~halt+name',
-  'railway~subway_entrance+name',
-  'railway~train_station_entrance+name',
-  'highway~pedestrian+area~yes+name',
-  'place~square+name',
-  'sport+name',
-  'natural+name',
-  'tourism+name',
-  'leisure+name',
-  'historic+name',
-  'man_made+name',
-  'landuse+name',
-  'waterway+name',
-  'aerialway+name',
-  'craft+name',
-  'military+name',
-  'aeroway~terminal+name',
-  'aeroway~aerodrome+name',
-  'aeroway~helipad+name',
-  'aeroway~airstrip+name',
-  'aeroway~heliport+name',
-  'aeroway~areodrome+name',
-  'aeroway~spaceport+name',
-  'aeroway~landing_strip+name',
-  'aeroway~airfield+name',
-  'aeroway~airport+name',
-  'brand+name',
-  'healthcare+name'
-];
+ layers: keyed by output layer name. order determines classifier priority (first match wins).
+         add custom layers here (e.g. river, protected_area) to extend the importer.
+**/
 
-module.exports = {tags,venue_tags};
+module.exports = {
+  addressTags: [
+    'addr:housenumber+addr:street',
+    'addr:housenumber+addr:place'  // @ref https://github.com/pelias/pelias/issues/787#issuecomment-477137803
+  ],
+  layers: {
+    venue: {
+      tags: [
+        'amenity+name',
+        'building+name',
+        'shop+name',
+        'office+name',
+        'public_transport+name',
+        'cuisine+name',
+        'railway~station+name',
+        'railway~tram_stop+name',
+        'railway~halt+name',
+        'railway~subway_entrance+name',
+        'railway~train_station_entrance+name',
+        'highway~pedestrian+area~yes+name',
+        'place~square+name',
+        'sport+name',
+        'natural+name',
+        'tourism+name',
+        'leisure+name',
+        'historic+name',
+        'man_made+name',
+        'landuse+name',
+        'waterway+name',
+        'aerialway+name',
+        'craft+name',
+        'military+name',
+        'aeroway~terminal+name',
+        'aeroway~aerodrome+name',
+        'aeroway~helipad+name',
+        'aeroway~airstrip+name',
+        'aeroway~heliport+name',
+        'aeroway~areodrome+name',
+        'aeroway~spaceport+name',
+        'aeroway~landing_strip+name',
+        'aeroway~airfield+name',
+        'aeroway~airport+name',
+        'brand+name',
+        'healthcare+name'
+      ]
+    }
+  }
+};

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "pelias-openstreetmap",
-  "version": "8.3.0",
+  "version": "8.3.2",
   "engines": {
     "node": ">=10.0.0"
   },
   "author": "pelias",
   "description": "Pelias openstreetmap utilities",
-  "homepage": "https://github.com/mapzen/pelias-openstreetmap",
+  "homepage": "https://github.com/pelias/openstreetmap",
   "license": "MIT",
   "main": "index.js",
   "scripts": {
@@ -22,7 +22,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/mapzen/pelias-openstreetmap.git"
+    "url": "https://github.com/pelias/openstreetmap.git"
   },
   "keywords": [
     "pelias",
@@ -31,7 +31,7 @@
     "elasticsearch"
   ],
   "bugs": {
-    "url": "https://github.com/mapzen/pelias-openstreetmap/issues"
+    "url": "https://github.com/pelias/openstreetmap/issues"
   },
   "dependencies": {
     "async": "^3.0.1",

package/schema.js

@@ -21,7 +21,14 @@ module.exports = Joi.object().keys({
       download: Joi.array().items(Joi.object().keys({
         sourceURL: Joi.string()
       }).requiredKeys('sourceURL').unknown(true)),
-      deduplicate: Joi.boolean()
+      deduplicate: Joi.boolean(),
+      addressTags: Joi.array().items(Joi.string()),
+      layers: Joi.object().pattern(
+        Joi.string(),
+        Joi.object().keys({
+          tags: Joi.array().items(Joi.string())
+        })
+      )
     }).requiredKeys('datapath', 'leveldbpath', 'import').unknown(true)
   }).requiredKeys('openstreetmap').unknown(true)
 }).requiredKeys('imports').unknown(true);

package/stream/document_constructor.js

@@ -8,6 +8,11 @@ const through = require('through2');
 const Document = require('pelias-model').Document;
 const peliasLogger = require( 'pelias-logger' ).get( 'openstreetmap' );
 const _ = require('lodash');
+const settings = require('pelias-config').generate(require('../schema'));
+const resolveFeatures = require('../util/resolveFeatures');
+const { classify } = require('../util/layerClassifier');
+
+const { layers } = resolveFeatures(settings);
 
 module.exports = function(){
 
@@ -19,8 +24,10 @@ module.exports = function(){
       }
       var uniqueId = [ item.type, item.id ].join('/');
 
-      // we need to assume it will be a venue and later if it turns out to be an address it will get changed
-      var doc = new Document( 'openstreetmap', 'venue', uniqueId );
+      // classify the layer from OSM tags; address_extractor assigns the address
+      // layer downstream — the classifier only covers named layers (venue, etc.)
+      var layer = classify(item.tags || {}, layers);
+      var doc = new Document( 'openstreetmap', layer, uniqueId );
 
       // Set latitude / longitude
       if( item.hasOwnProperty('lat') && item.hasOwnProperty('lon') ){

package/stream/importPipeline.js

@@ -13,6 +13,7 @@ streams.tagMapper = require('./tag_mapper');
 streams.addressesWithoutStreet = require('./addresses_without_street');
 streams.adminLookup = require('pelias-wof-admin-lookup').create;
 streams.addressExtractor = require('./address_extractor');
+streams.venueFilter = require('./venueFilter');
 streams.categoryMapper = require('./category_mapper');
 streams.addendumMapper = require('./addendum_mapper');
 streams.popularityMapper = require('./popularity_mapper');
@@ -26,6 +27,7 @@ streams.import = function(){
     .pipe( streams.addressesWithoutStreet() )
     .pipe( streams.tagMapper() )
     .pipe( streams.addressExtractor() )
+    .pipe( streams.venueFilter() )
     .pipe( streams.blacklistStream() )
     .pipe( streams.categoryMapper( categoryDefaults ) )
     .pipe( streams.addendumMapper() )

package/stream/pbf.js

@@ -7,7 +7,7 @@
 var fs = require('fs'),
     pbf2json = require('pbf2json'),
     settings = require('pelias-config').generate(require('../schema')),
-    features = require('../config/features'),
+    resolveFeatures = require('../util/resolveFeatures'),
     path = require('path');
 
 // Create a new pbf parser stream
@@ -46,11 +46,11 @@ function config(opts){
   if(!opts.tags){
     // check if we import venues
     opts.importVenues = settings.imports.openstreetmap.import[0].importVenues;
-    if(opts.importVenues){
-      opts.tags = features.tags.concat(features.venue_tags);
-    } else {
-      opts.tags = features.tags;
-    }
+    const features = resolveFeatures(settings);
+    const layerTags = opts.importVenues === false
+      ? []
+      : Object.values(features.layers).flatMap(l => l.tags ?? []);
+    opts.tags = features.addressTags.concat(layerTags);
   }
   return opts;
 }

package/stream/venueFilter.js

@@ -0,0 +1,30 @@
+'use strict';
+
+/**
+ * When importVenues is false, drop any record that is not in the address layer.
+ * This prevents named POIs that also have address tags from leaking through as
+ * venue records when venue import is disabled.
+ *
+ * Note: the preferred way to control which layers are imported is now to set
+ * `imports.openstreetmap.layers` in pelias.json. Setting `importVenues: false`
+ * is equivalent to `layers: {}` and is retained for backward compatibility.
+ */
+
+const through = require('through2');
+const settings = require('pelias-config').generate(require('../schema'));
+
+module.exports = function() {
+  const importVenues = settings.imports.openstreetmap.import[0].importVenues;
+
+  // passthrough when venues are enabled (default)
+  if (importVenues !== false) {
+    return through.obj();
+  }
+
+  return through.obj(function(doc, enc, next) {
+    if (doc.getLayer() === 'address') {
+      this.push(doc);
+    }
+    next();
+  });
+};

package/util/layerClassifier.js

@@ -0,0 +1,43 @@
+'use strict';
+
+/**
+ * Test a single pbf2json tag condition against an OSM tags object.
+ * Supports 'key' (key must exist) or 'key~value' (key must equal value).
+ */
+function conditionMatchesTags(condition, tags) {
+  const tildeIdx = condition.indexOf('~');
+  if (tildeIdx === -1) {
+    return Object.prototype.hasOwnProperty.call(tags, condition);
+  }
+  const key = condition.slice(0, tildeIdx);
+  const value = condition.slice(tildeIdx + 1);
+  return tags[key] === value;
+}
+
+/**
+ * Test whether a full pbf2json pattern string matches a tags object.
+ * Pattern conditions are separated by '+' and all must match (AND logic).
+ */
+function patternMatchesTags(pattern, tags) {
+  return pattern.split('+').every(cond => conditionMatchesTags(cond, tags));
+}
+
+/**
+ * Classify an OSM tags object against the features config.
+ * Iterates layers in declaration order and returns the first matching layer.
+ * Falls back to 'venue' if no patterns match.
+ *
+ * @param {Object} tags     - OSM tags from a pbf2json record
+ * @param {Object} features - the features.js keyed-by-layer export
+ * @returns {string} layer name
+ */
+function classify(tags, features) {
+  for (const [layer, config] of Object.entries(features)) {
+    if (Array.isArray(config.tags) && config.tags.some(p => patternMatchesTags(p, tags))) {
+      return layer;
+    }
+  }
+  return 'venue';
+}
+
+module.exports = { patternMatchesTags, classify };

package/util/resolveFeatures.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const defaults = require('../config/features');
+
+/**
+ * Resolve the effective features config by merging pelias.json overrides
+ * with the defaults from config/features.js.
+ *
+ * If `imports.openstreetmap.layers` is set in pelias.json it fully replaces
+ * the default layer set. Same for `addressTags`. This lets operators add
+ * custom layers (e.g. protected_area) or restrict which tags are extracted
+ * without touching the source code.
+ *
+ * @param {Object} settings - result of pelias-config generate()
+ * @returns {{ addressTags: string[], layers: Object }}
+ */
+function resolveFeatures(settings) {
+  const osmConfig = (settings && settings.imports && settings.imports.openstreetmap) || {};
+  return {
+    addressTags: osmConfig.addressTags || defaults.addressTags,
+    layers:      osmConfig.layers      || defaults.layers
+  };
+}
+
+module.exports = resolveFeatures;