pelias-openstreetmap 5.13.0

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "pelias-openstreetmap",
+  "version": "5.13.0",
+  "engines": {
+    "node": ">=10.0.0"
+  },
+  "author": "pelias",
+  "description": "Pelias openstreetmap utilities",
+  "homepage": "https://github.com/mapzen/pelias-openstreetmap",
+  "license": "MIT",
+  "main": "index.js",
+  "scripts": {
+    "download": "./bin/download",
+    "coverage": "node_modules/.bin/istanbul cover test/run.js",
+    "end-to-end": "export NODE_ENV=test && node test/end-to-end.js;",
+    "lint": "jshint .",
+    "start": "./bin/start",
+    "test": "npm run units",
+    "ci": "node ./test/ci-config.js && npm test && npm run end-to-end",
+    "units": "./bin/test",
+    "validate": "npm ls"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mapzen/pelias-openstreetmap.git"
+  },
+  "keywords": [
+    "pelias",
+    "stream",
+    "openstreetmap",
+    "elasticsearch"
+  ],
+  "bugs": {
+    "url": "https://github.com/mapzen/pelias-openstreetmap/issues"
+  },
+  "dependencies": {
+    "async": "^3.0.1",
+    "combined-stream": "^1.0.5",
+    "extend": "^3.0.0",
+    "is-object": "^1.0.1",
+    "iso-639-3": "^1.0.0",
+    "joi": "^14.0.0",
+    "lodash": "^4.17.4",
+    "pbf2json": "^6.9.0",
+    "pelias-blacklist-stream": "^1.0.0",
+    "pelias-config": "^4.12.0",
+    "pelias-dbclient": "^2.13.0",
+    "pelias-logger": "^1.2.1",
+    "pelias-model": "^9.1.1",
+    "pelias-wof-admin-lookup": "^7.4.1",
+    "through2": "^3.0.0",
+    "through2-sink": "^1.0.0"
+  },
+  "devDependencies": {
+    "colors": "^1.1.2",
+    "deep-diff": "^1.0.0",
+    "istanbul": "^0.4.3",
+    "jshint": "^2.10.3",
+    "naivedb": "^1.0.7",
+    "pelias-mock-logger": "^1.0.1",
+    "precommit-hook": "^3.0.0",
+    "proxyquire": "^2.0.0",
+    "stream-mock": "^2.0.3",
+    "taginfo": "^1.0.1",
+    "tap-spec": "^5.0.0",
+    "tape": "^5.0.0"
+  },
+  "pre-commit": [
+    "lint",
+    "validate",
+    "test"
+  ],
+  "release": {
+    "branch": "master",
+    "success": []
+  }
+}

package/schema/address_karlsruhe.js

@@ -0,0 +1,25 @@
+
+/**
+  Attempt to map OSM address tags to the Pelias address schema.
+
+  On the left is the OSM tag name, on the right is corresponding
+  doc.address_parts key for which it should be mapped to.
+
+  eg. tags['addr:street'] -> doc.address_parts['street']
+
+  @ref: http://wiki.openstreetmap.org/wiki/Karlsruhe_Schema
+**/
+
+var KARLSRUHE_SCHEMA = {
+  'addr:housename':     'name',
+  'addr:housenumber':   'number',
+  'addr:street':        'street',
+  'addr:postcode':      'zip'
+
+  // @ref: https://github.com/pelias/model/pull/13
+  // 'addr:state':         'state',
+  // 'addr:city':          'city',
+  // 'addr:country':       'country'
+};
+
+module.exports = KARLSRUHE_SCHEMA;

package/schema/address_naptan.js

@@ -0,0 +1,17 @@
+
+/**
+  Attempt to map OSM address tags to the Pelias address schema.
+
+  On the left is the OSM tag name, on the right is corresponding
+  doc.address key for which it should be mapped to.
+
+  eg. tags['naptan:Street'] -> doc.address_parts['street']
+
+  @ref: http://wiki.openstreetmap.org/wiki/NaPTAN
+**/
+
+var NAPTAN_SCHEMA = {
+  'naptan:Street': 'street'
+};
+
+module.exports = NAPTAN_SCHEMA;

package/schema/address_osm.js

@@ -0,0 +1,17 @@
+
+/**
+  Attempt to map OSM address tags to the Pelias address schema.
+
+  On the left is the OSM tag name, on the right is corresponding
+  doc.address key for which it should be mapped to.
+
+  eg. tags['postal_code'] -> doc.address_parts['zip']
+
+  @ref: http://wiki.openstreetmap.org/wiki/Key:postal_code
+**/
+
+var OSM_SCHEMA = {
+  'postal_code': 'zip'
+};
+
+module.exports = OSM_SCHEMA;

package/schema/address_tiger.js

@@ -0,0 +1,18 @@
+
+/**
+  Attempt to map OSM TIGER tags to the Pelias address schema.
+
+  On the left is the OSM tag name, on the right is corresponding
+  doc.address key for which it should be mapped to.
+
+  eg. tags['tiger:zip_left'] -> doc.address_parts['zip']
+
+  @ref: http://wiki.openstreetmap.org/wiki/TIGER_to_OSM_Attribute_Map
+**/
+
+var TIGER_SCHEMA = {
+  'tiger:zip_left': 'zip',
+  'tiger:zip_right': 'zip'
+};
+
+module.exports = TIGER_SCHEMA;

package/schema/name_osm.js

@@ -0,0 +1,51 @@
+
+/**
+  Attempt to map OSM name tags to the Pelias naming schema.
+
+  On the left is the OSM tag name, on the right is corresponding
+  doc.name key for which it should be mapped to.
+
+  eg. tags['alt_name'] -> doc.name['alternative']
+
+  The value 'default' is special, it defines the name used by default
+  for elasticsearch queries, it has the unique ability to specify multiple
+  keys with a value of 'default'.
+
+  When multiple keys have the value 'default' then they are considered
+  as aliases of the default field.
+
+  The '_primary' property defined below defines which of those aliases
+  is considered the 'primary default name' for label generation.
+
+  No values other than 'default' should be specified more than once.
+
+  @ref: http://wiki.openstreetmap.org/wiki/Key:name
+  @ref: http://wiki.openstreetmap.org/wiki/Names
+**/
+
+var OSM_NAMING_SCHEMA = {
+  'name':             'default',
+  'loc_name':         'default',
+  'alt_name':         'default',
+  'short_name':       'default',
+
+  // note: these aliases are currently disabled because they are not being used when querying
+  // 'int_name':         'international',
+  // 'nat_name':         'national',
+  // 'official_name':    'official',
+  // 'old_name':         'old',
+  // 'reg_name':         'regional',
+  // 'sorting_name':     'sorting'
+};
+
+// this property is considered the 'primary name'
+// for label generation, the others are considered
+// secondary or 'aliases'.
+Object.defineProperty(OSM_NAMING_SCHEMA, '_primary', {
+  value: 'name',
+  enumerable: false,
+  configurable: false,
+  writable: false
+});
+
+module.exports = OSM_NAMING_SCHEMA;

package/schema.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const Joi = require('joi');
+
+// Schema Configuration
+// datapath: string (required)
+// leveldbpath: string (required)
+// import: array of objects containing filename (optional)
+// importVenues: boolean (optional)
+// download: array of objects containing sourceURL (optional)
+// deduplicate: boolean (optional)
+module.exports = Joi.object().keys({
+  imports: Joi.object().keys({
+    openstreetmap: Joi.object().keys({
+      datapath: Joi.string(),
+      leveldbpath: Joi.string(),
+      import: Joi.array().items(Joi.object().keys({
+        filename: Joi.string(),
+        importVenues: Joi.boolean().default(true).truthy('yes').falsy('no').insensitive(true)
+      }).requiredKeys('filename').unknown(true)),
+      download: Joi.array().items(Joi.object().keys({
+        sourceURL: Joi.string()
+      }).requiredKeys('sourceURL').unknown(true)),
+      deduplicate: Joi.boolean()
+    }).requiredKeys('datapath', 'leveldbpath', 'import').unknown(true)
+  }).requiredKeys('openstreetmap').unknown(true)
+}).requiredKeys('imports').unknown(true);

package/stream/addendum_mapper.js

@@ -0,0 +1,71 @@
+/**
+  The addendum mapper is responsible for adding interesting metadata
+  as an 'addendum' to the record.
+
+  @see: https://github.com/pelias/api/pull/1255
+**/
+
+const through = require('through2');
+const peliasLogger = require('pelias-logger').get('openstreetmap');
+const whitelist = [
+  'wheelchair', // Wheelchair accessibility
+  'iata', // IATA airport codes
+  'icao', // ICAO airport codes
+  'wikidata', // Wikidata concordance
+  'wikipedia', // Wikipedia concordance
+  'operator', // Operator name
+  'brand', // Brand name
+  'website', // Website URL
+  'phone', // Telephone number
+  'opening_hours', // Opening hours
+
+  // COVID-19
+  'opening_hours:covid19',
+  'delivery:covid19',
+  'safety:mask:covid19'
+];
+
+module.exports = function(){
+
+  return through.obj(( doc, enc, next ) => {
+
+    try {
+
+      // skip records with no tags
+      let tags = doc.getMeta('tags');
+      if( !tags ){
+        return next( null, doc );
+      }
+
+      let addendum = {};
+
+      // iterate over mapping
+      whitelist.forEach(key => {
+
+        // check each whitelist key against document tags
+        if( !tags.hasOwnProperty( key ) ){ return; }
+
+        // set addendum key
+        let value = tags[key].trim();
+        if( !!value.length ){
+          addendum[ key ] = value;
+        }
+      });
+
+      // set document addendum
+      if( !!Object.keys( addendum ).length ){
+        doc.setAddendum( 'osm', addendum );
+      }
+    }
+
+    catch( e ){
+      peliasLogger.error( 'addendum_mapper error' );
+      peliasLogger.error( e.stack );
+      peliasLogger.error( JSON.stringify( doc, null, 2 ) );
+    }
+
+    return next( null, doc );
+
+  });
+
+};

package/stream/address_extractor.js

@@ -0,0 +1,131 @@
+
+/**
+  The address extractor is responsible for cloning documents where a valid address
+  data exists.
+
+  The hasValidAddress() function examines the address property which was populated
+  earier in the pipeline by the osm.tag.mapper stream and therefore MUST come after
+  that stream in the pipeline or it will fail to find any address information.
+
+  There are a few different outcomes for this stream depending on the data contained
+  in each individual document, the result can be, 0, 1 or 2 documents being emitted.
+
+  In the case of the document missing a valid doc.name.default string then it is not
+  considered to be a point-of-interest in it's own right, it will be discarded.
+
+  In the case where the document contains BOTH a valid house number & street name we
+  consider this record to be an address in it's own right and we clone that record,
+  duplicating the data across to the new doc instance while adjusting it's id and layer.
+
+  In a rare case it is possible that the record contains neither a valid name nor a valid
+  address. If this case in encountered then the parser should be modified so these records
+  are no longer passed down the pipeline; as they will simply be discarded because they are
+  not searchable.
+**/
+
+var through = require('through2');
+var isObject = require('is-object');
+var extend = require('extend');
+var peliasLogger = require( 'pelias-logger' ).get( 'openstreetmap' );
+var Document = require('pelias-model').Document;
+
+function hasValidAddress( doc ){
+  if( !isObject( doc ) ){ return false; }
+  if( !isObject( doc.address_parts ) ){ return false; }
+  if( 'string' !== typeof doc.address_parts.number ){ return false; }
+  if( 'string' !== typeof doc.address_parts.street ){ return false; }
+  if( !doc.address_parts.number.length ){ return false; }
+  if( !doc.address_parts.street.length ){ return false; }
+  return true;
+}
+
+module.exports = function(){
+
+  var stream = through.obj( function( doc, enc, next ) {
+    var isNamedPoi = !!doc.getName('default');
+    var isAddress = hasValidAddress( doc );
+
+    // accept semi-colon delimited house numbers
+    // ref: https://github.com/pelias/openstreetmap/issues/21
+    var streetNumbers = (doc.getAddress('number') || '').split(';').map(Function.prototype.call, String.prototype.trim);
+
+    // create a new record for street addresses
+    if( isAddress ){
+      var record;
+
+      streetNumbers.forEach( function( streetno, i ){
+
+        try {
+          var newid = [ doc.getSourceId() ];
+          if( i > 0 ){
+            newid.push( i );
+            peliasLogger.debug('[address_extractor] found multiple house numbers: ', streetNumbers);
+          }
+
+          // copy data to new document
+          record = new Document( 'openstreetmap', 'address', newid.join('/') )
+            .setName( 'default', streetno + ' ' + doc.address_parts.street )
+            .setCentroid( doc.getCentroid() );
+
+          // copy all address properties
+          setProperties( record, doc );
+
+          // ensure that only a single address number is used
+          record.setAddress('number', streetno);
+        }
+
+        catch( e ){
+          peliasLogger.error( 'address_extractor error' );
+          peliasLogger.error( e.stack );
+          peliasLogger.error( JSON.stringify( doc, null, 2 ) );
+        }
+
+        if( record !== undefined ){
+          // copy meta data (but maintain the id assigned above)
+          record._meta = extend( true, {}, doc._meta, { id: record.getId() } );
+          this.push( record );
+        }
+        else {
+          peliasLogger.error( '[address_extractor] failed to push address downstream' );
+        }
+
+      }, this);
+
+    }
+
+    // forward doc downstream if it's a POI in its own right
+    // note: this MUST be below the address push()
+    if( isNamedPoi ){
+
+      // in the case of multiple house numbers, only use the first one for the venue
+      if (streetNumbers.length > 1) {
+        doc.setAddress('number', streetNumbers[0]);
+      }
+
+      this.push( doc );
+    }
+
+    return next();
+
+  });
+
+  // catch stream errors
+  stream.on( 'error', peliasLogger.error.bind( peliasLogger, __filename ) );
+
+  return stream;
+};
+
+// properties to map from the osm record to the pelias doc
+var addrProps = [ 'name', 'number', 'street', 'zip' ];
+
+// call document setters and ignore non-fatal errors
+function setProperties( record, doc ){
+  addrProps.forEach( function ( prop ){
+    try {
+      record.setAddress( prop, doc.getAddress( prop ) );
+    } catch ( ex ) {}
+  });
+}
+
+// export for testing
+module.exports.hasValidAddress = hasValidAddress;

package/stream/category_mapper.js

@@ -0,0 +1,77 @@
+
+/**
+  The category mapper is responsible for checking each documents tags
+  against a predefined taxonomy mapping file.
+
+  For each openstreetmap 'feature' that matches the mapping file we
+  create a new document 'category' by calling `doc.addCategory( category );`.
+
+  This idea is to classify places-of-interest in order to facilitate the
+  building of industry and interest specific geocoders.
+
+  @see: https://github.com/pelias/pelias/wiki/Taxonomy-v1
+  @see: ./config/category_map.js
+**/
+
+const _ = require('lodash');
+const through = require('through2');
+const peliasLogger = require('pelias-logger').get('openstreetmap');
+
+module.exports = function( mapping ){
+
+  return through.obj( function( doc, enc, next ){
+
+    try {
+
+      // do not categorize addresses
+      if( doc.getLayer().match('address') ){
+        return next( null, doc );
+      }
+
+      // skip records with no tags
+      var tags = doc.getMeta('tags');
+      if( !tags ){
+        return next( null, doc );
+      }
+
+      // iterate over mapping
+      _.each(mapping, (osmTagValues, osmTagKey) => {
+
+        // check each mapping key against document tags
+        if( !tags.hasOwnProperty( osmTagKey ) ){ return; }
+
+        // handle mapping wildcards
+        if( osmTagValues.hasOwnProperty('*') ){
+          addCategories( doc, osmTagValues['*'] );
+        }
+
+        // handle regular features
+        _.each(osmTagValues, (categories, osmTagValue) => {
+          if( '*' === osmTagValue ){ return; }
+          if( tags[osmTagKey] === osmTagValue ){
+            addCategories( doc, categories );
+          }
+        });
+      });
+    }
+
+    catch( e ){
+      peliasLogger.error( 'category_mapper error' );
+      peliasLogger.error( e.stack );
+      peliasLogger.error( JSON.stringify( doc, null, 2 ) );
+    }
+
+    return next( null, doc );
+
+  });
+
+};
+
+function addCategories( doc, categories ){
+  if( !Array.isArray( categories ) ){
+    categories = [categories];
+  }
+  categories.forEach( function( category ){
+    doc.addCategory( category );
+  });
+}

package/stream/document_constructor.js

@@ -0,0 +1,76 @@
+
+/**
+  The document constructor is responsible for mapping input data from the parser
+  in to model.Document() objects which the rest of the pipeline expect to consume.
+**/
+
+const through = require('through2');
+const Document = require('pelias-model').Document;
+const peliasLogger = require( 'pelias-logger' ).get( 'openstreetmap' );
+const _ = require('lodash');
+
+module.exports = function(){
+
+  var stream = through.obj( function( item, enc, next ) {
+
+    try {
+      if (!item.type || ! item.id) {
+        throw new Error('doc without valid id or type');
+      }
+      var uniqueId = [ item.type, item.id ].join('/');
+
+      // we need to assume it will be a venue and later if it turns out to be an address it will get changed
+      var doc = new Document( 'openstreetmap', 'venue', uniqueId );
+
+      // Set latitude / longitude
+      if( item.hasOwnProperty('lat') && item.hasOwnProperty('lon') ){
+        doc.setCentroid({
+          lat: item.lat,
+          lon: item.lon
+        });
+      }
+
+      // Set latitude / longitude (for ways where the centroid has been precomputed)
+      else if( item.hasOwnProperty('centroid') ){
+        if( item.centroid.hasOwnProperty('lat') && item.centroid.hasOwnProperty('lon') ){
+          doc.setCentroid({
+            lat: item.centroid.lat,
+            lon: item.centroid.lon
+          });
+        }
+      }
+
+      // set bounding box
+      if( _.isPlainObject(item.bounds) ){
+        doc.setBoundingBox({
+          upperLeft: {
+            lat: parseFloat(item.bounds.n),
+            lon: parseFloat(item.bounds.w)
+          },
+          lowerRight: {
+            lat: parseFloat(item.bounds.s),
+            lon: parseFloat(item.bounds.e)
+          }
+        });
+      }
+
+      // Store osm tags as a property inside _meta
+      doc.setMeta( 'tags', item.tags || {} );
+
+      // Push instance of Document downstream
+      this.push( doc );
+    }
+
+    catch( e ){
+      peliasLogger.error( 'error constructing document model', e.stack );
+    }
+
+    return next();
+
+  });
+
+  // catch stream errors
+  stream.on( 'error', peliasLogger.error.bind( peliasLogger, __filename ) );
+
+  return stream;
+};

package/stream/importPipeline.js

@@ -0,0 +1,36 @@
+var categoryDefaults = require('../config/category_map');
+
+var streams = {};
+
+streams.config = {
+  categoryDefaults: categoryDefaults
+};
+
+streams.pbfParser = require('./multiple_pbfs').create;
+streams.docConstructor = require('./document_constructor');
+streams.blacklistStream = require('pelias-blacklist-stream');
+streams.tagMapper = require('./tag_mapper');
+streams.adminLookup = require('pelias-wof-admin-lookup').create;
+streams.addressExtractor = require('./address_extractor');
+streams.categoryMapper = require('./category_mapper');
+streams.addendumMapper = require('./addendum_mapper');
+streams.popularityMapper = require('./popularity_mapper');
+streams.dbMapper = require('pelias-model').createDocumentMapperStream;
+streams.elasticsearch = require('pelias-dbclient');
+
+// default import pipeline
+streams.import = function(){
+  streams.pbfParser()
+    .pipe( streams.docConstructor() )
+    .pipe( streams.tagMapper() )
+    .pipe( streams.addressExtractor() )
+    .pipe( streams.blacklistStream() )
+    .pipe( streams.categoryMapper( categoryDefaults ) )
+    .pipe( streams.addendumMapper() )
+    .pipe( streams.popularityMapper() )
+    .pipe( streams.adminLookup() )
+    .pipe( streams.dbMapper() )
+    .pipe( streams.elasticsearch({name: 'openstreetmap'}) );
+};
+
+module.exports = streams;

package/stream/multiple_pbfs.js

@@ -0,0 +1,25 @@
+var combinedStream = require('combined-stream');
+var pbf = require('./pbf');
+var path = require('path');
+var logger = require('pelias-logger').get('openstreetmap');
+
+function createCombinedStream(){
+  var fullStream = combinedStream.create();
+  var defaultPath= require('pelias-config').generate().imports.openstreetmap;
+
+  defaultPath.import.forEach(function( importObject){
+    var conf = {
+      file: path.join(defaultPath.datapath, importObject.filename),
+      leveldb: defaultPath.leveldbpath,
+      importVenues: importObject.importVenues
+    };
+    fullStream.append(function(next){
+      logger.info('Creating read stream for: ' + conf.file);
+      next(pbf.parser(conf));
+    });
+  });
+
+  return fullStream;
+}
+
+module.exports.create = createCombinedStream;