pelias-openstreetmap 5.13.0

package/stream/pbf.js

@@ -0,0 +1,75 @@
+
+/**
+  The pbf parser is responsible for configuring and executing a pbf parser and
+  returning a valid readable stream.
+**/
+
+var fs = require('fs'),
+    pbf2json = require('pbf2json'),
+    settings = require('pelias-config').generate(require('../schema')),
+    features = require('../config/features'),
+    path = require('path');
+
+// Create a new pbf parser stream
+function createPbfStream(opts){
+
+  var conf = config(opts);
+
+  validatePath( conf.file, 'failed to stat pbf file: ' + conf.file );
+  validatePath( conf.leveldb, 'failed to stat leveldb path: ' + conf.leveldb );
+  validateTags( conf.tags );
+
+  return pbf2json.createReadStream(conf);
+}
+
+// Generate configuration options for pbf2json, apply default
+// configurations where not explicitly specified.
+function config(opts){
+  if (!opts){
+    opts = {};
+  }
+
+  // Use datapath setting from your config file
+  // @see: github://pelias/config for more info.
+  if(!opts.file){
+     var filename = settings.imports.openstreetmap.import[0].filename;
+     opts.file = path.join(settings.imports.openstreetmap.datapath, filename);
+  }
+
+  // Use leveldb setting from your config file
+  // @see: github://pelias/config for more info.
+  if(!opts.leveldb){
+    opts.leveldb = settings.imports.openstreetmap.leveldbpath;
+  }
+
+  // Use default parser tags
+  if(!opts.tags){
+    // check if we import venues
+    opts.importVenues = settings.imports.openstreetmap.import[0].importVenues;
+    if(opts.importVenues){
+      opts.tags = features.tags.concat(features.venue_tags);
+    } else {
+      opts.tags = features.tags;
+    }
+  }
+  return opts;
+}
+
+// Check path exists
+function validatePath( path, message ){
+  try {
+    fs.statSync( path );
+  } catch( e ){
+    throw new Error( message );
+  }
+}
+
+// Validate the tag list
+function validateTags( tags ){
+  if( !Array.isArray(tags) || !tags.length ) {
+    throw new Error( 'invalid tags' );
+  }
+}
+
+module.exports.config = config;
+module.exports.parser = createPbfStream;

package/stream/popularity_mapper.js

@@ -0,0 +1,80 @@
+/**
+  The popularity mapper is responsible for generating a 'popularity'
+  value by inspecting OSM tags.
+
+  Disused and abandoned places are given a strong negative score.
+  If the popularity score is less than zero then the document is discarded.
+
+  Feel free to make changes to this mapping file!
+**/
+
+const _ = require('lodash');
+const through = require('through2');
+const peliasLogger = require('pelias-logger').get('openstreetmap');
+const peliasConfig = require('pelias-config').generate();
+const mapping = require('../config/popularity_map');
+
+module.exports = function(){
+
+  return through.obj(( doc, enc, next ) => {
+
+    try {
+
+      // only map venues
+      if( !['venue', 'address'].includes(doc.getLayer()) ){
+        return next(null, doc);
+      }
+
+      // skip records with no tags
+      let tags = doc.getMeta('tags');
+      if( !tags ){
+        return next( null, doc );
+      }
+
+      // default popularity
+      let popularity = doc.getPopularity() || 0;
+
+      // apply scores from config
+      _.each(mapping, (osmTagScores, osmTagKey) => {
+        if( tags.hasOwnProperty( osmTagKey ) ){
+          // global score for the tag
+          if( osmTagScores._score ){
+            popularity += osmTagScores._score;
+          }
+          // individual scores for specific values
+          _.each(osmTagScores, (osmSubTagScores, osmTagValue) => {
+            if( osmTagValue === '_score' ){ return; }
+            if( !osmSubTagScores._score ){ return; }
+            if (_.get(tags, osmTagKey, '').trim().toLowerCase() === osmTagValue ){
+              popularity += osmSubTagScores._score;
+            }
+          });
+        }
+      });
+
+      // addresses with a popularity score GTE 10000 receieve
+      // a popularity of 1000, all others get a popularity of 0.
+      if ( doc.getLayer() === 'address' ){
+        popularity = (popularity >= 10000) ? 1000 : 0;
+      }
+
+      // set document popularity if it is greater than zero
+      if( popularity > 0 ){ doc.setPopularity( popularity ); }
+
+      // discard places with a negative popularity
+      else if( popularity < 0 && peliasConfig.get('imports.openstreetmap.removeDisusedVenues') === true ){
+        peliasLogger.warn(`removing record ${doc.getGid()} (${doc.getName('default')}) with popularity ${popularity}`);
+        return next();
+      }
+    }
+
+    catch( e ){
+      peliasLogger.error( 'popularity_mapper error' );
+      peliasLogger.error( e.stack );
+      peliasLogger.error( JSON.stringify( doc, null, 2 ) );
+    }
+
+    return next( null, doc );
+  });
+
+};

package/stream/stats.js

@@ -0,0 +1,47 @@
+
+/**
+  The stats module is useful for recording stats while developing new streams.
+
+  @usage:
+
+    tap.pipe( stream1 )
+      .pipe( stats.proxy('stream1 -> stream2') )
+      .pipe( stream2 )
+      .pipe( stats.proxy('stream2 -> stream3') )
+      .pipe( stream3 )
+      .pipe( stats.proxy('stream3 -> sink') )
+      .pipe( sink )
+
+  you can then get throughput statistics in your terminal to identify bottlenecks:
+
+    var peliasLogger = require( 'pelias-logger' ).get( 'openstreetmap' );
+    setInterval( peliasLogger.log.bind( peliasLogger, stats.metrics ), 1000 );
+**/
+
+var through = require('through2');
+
+function Stats(){
+  this.reset();
+}
+
+Stats.prototype.reset = function(){
+  this.metrics = {};
+};
+
+Stats.prototype.proxy = function( title ){
+  if( !this.metrics.hasOwnProperty( title ) ){
+    this.metrics[ title ] = 0;
+  }
+
+  var self = this;
+
+  var proxy = through.obj( function( item, enc, next ) {
+    self.metrics[ title ]++;
+    this.push( item );
+    return next();
+  });
+
+  return proxy;
+};
+
+module.exports = new Stats();

package/stream/stringify.js

@@ -0,0 +1,9 @@
+var through = require('through2');
+
+// convenience function for converting object streams back to strings
+var stringify = through.obj( function( data, enc, next ){
+  this.push( JSON.stringify( data, null, 2 ), 'utf-8' );
+  next();
+});
+
+module.exports = stringify;

package/stream/tag_mapper.js

@@ -0,0 +1,168 @@
+
+/**
+  The tag mapper is responsible for mapping OSM tag information in to the
+  document model, using a variety of different schemas found in /schema/*.
+**/
+
+const _ = require('lodash');
+const through = require('through2');
+const peliasLogger = require('pelias-logger').get('openstreetmap');
+
+var LOCALIZED_NAME_KEYS = require('../config/localized_name_keys');
+var NAME_SCHEMA = require('../schema/name_osm');
+var ADDRESS_SCHEMA = _.merge( {},
+  require('../schema/address_tiger'),
+  require('../schema/address_osm'),
+  require('../schema/address_naptan'),
+  require('../schema/address_karlsruhe')
+);
+
+module.exports = function(){
+
+  var stream = through.obj( function( doc, enc, next ) {
+
+    try {
+
+      // skip records with no tags
+      var tags = doc.getMeta('tags');
+      if( !tags ){
+        return next( null, doc );
+      }
+
+      // Unfortunately we need to iterate over every tag,
+      // so we only do the iteration once to save CPU.
+      _.each(tags, (value, key) => {
+
+        // Map localized names which begin with 'name:'
+        // @ref: http://wiki.openstreetmap.org/wiki/Namespace#Language_code_suffix
+        var suffix = getNameSuffix( key );
+        if( suffix ){
+          var val1 = trim( value );
+          if( val1 ){
+            doc.setName( suffix, val1 );
+          }
+        }
+
+        // Map name data from our name mapping schema
+        else if( _.has(NAME_SCHEMA, key) ){
+          var val2 = trim( value );
+          if( val2 ){
+            if( key === NAME_SCHEMA._primary ){
+              doc.setName( NAME_SCHEMA[key], val2 );
+            } else if ( 'default' === NAME_SCHEMA[key] ) {
+              doc.setNameAlias( NAME_SCHEMA[key], val2 );
+            } else {
+              doc.setName( NAME_SCHEMA[key], val2 );
+            }
+          }
+        }
+
+        // Map address data from our address mapping schema
+        else if( _.has(ADDRESS_SCHEMA, key) ){
+          var val3 = trim( value );
+          if( val3 ){
+            let label = ADDRESS_SCHEMA[key];
+            doc.setAddress(label, normalizeAddressField(label, val3));
+          }
+        }
+      });
+
+      // Handle the case where no default name was set but there were
+      // other names which we could use as the default.
+      if( !doc.getName('default') ){
+
+        var defaultName =
+          _.get(tags, 'official_name') ||
+          _.get(tags, 'int_name') ||
+          _.get(tags, 'nat_name') ||
+          _.get(tags, 'reg_name') ||
+          doc.getName('en');
+
+        // use one of the preferred name tags listed above
+        if ( defaultName ){
+          doc.setName('default', defaultName);
+        }
+
+        // else try to use an available two-letter language name tag
+        else {
+          var keys = Object.keys(doc.name).filter(n => n.length === 2);
+
+          // unambiguous (there is only a single two-letter name tag)
+          if ( keys.length === 1 ){
+            doc.setName('default', doc.getName(keys[0]));
+          }
+
+          // note we do not handle ambiguous cases where the record contains >1
+          // two-letter name tags.
+          // see: https://github.com/pelias/openstreetmap/pull/498
+        }
+      }
+
+      // Import airport codes as aliases
+      if( tags.hasOwnProperty('aerodrome') || tags.hasOwnProperty('aeroway') ){
+        if( tags.hasOwnProperty('iata') ){
+          var iata = trim( tags.iata );
+          if( iata ){
+            doc.setNameAlias( 'default', iata );
+            doc.setNameAlias( 'default', `${iata} Airport` );
+          }
+        }
+      }
+    }
+
+    catch( e ){
+      peliasLogger.error( 'tag_mapper error' );
+      peliasLogger.error( e.stack );
+      peliasLogger.error( JSON.stringify( doc, null, 2 ) );
+    }
+
+    return next( null, doc );
+
+  });
+
+  // catch stream errors
+  stream.on( 'error', peliasLogger.error.bind( peliasLogger, __filename ) );
+
+  return stream;
+};
+
+// Clean string of leading/trailing junk chars
+function trim( str ){
+  return _.trim( str, '#$%^*<>-=_{};:",./?\t\n\' ' );
+}
+
+// extract name suffix, eg for 'name:EN' return 'en'
+// if not valid, return false.
+function getNameSuffix( tag ){
+
+  if( tag.length < 6 || tag.substr(0,5) !== 'name:' ){
+    return false;
+  }
+
+  // normalize suffix
+  var suffix = tag.substr(5).toLowerCase();
+
+  // check the suffix is in the localized key list
+  if( LOCALIZED_NAME_KEYS.indexOf(suffix) === -1 ){
+    return false;
+  }
+
+  return suffix;
+}
+
+// apply some very basic normalization
+// note: in the future we should consider doing something more advanced like:
+// https://github.com/pelias/openaddresses/pull/477
+function normalizeAddressField(key, value) {
+  if (key === 'street') {
+
+    // contract English diagonals
+    value = value
+      .replace(/\b(northwest)\b/i, 'NW')
+      .replace(/\b(northeast)\b/i, 'NE')
+      .replace(/\b(southwest)\b/i, 'SW')
+      .replace(/\b(southeast)\b/i, 'SE');
+  }
+
+  return value;
+}

package/util/download_data.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const _ = require('lodash');
+const async = require('async');
+const child_process = require('child_process');
+const fs = require('fs');
+const config = require( 'pelias-config' ).generate(require('../schema'));
+const logger = require('pelias-logger').get('openstreetmap-download');
+
+if (require.main === module) {
+  download((err) => {
+    if (err) {
+      logger.error('Failed to download data', err);
+      process.exit(1);
+    }
+    logger.info('All done!');
+  });
+}
+
+function download(callback) {
+  let sources;
+
+  // if no download sources are specified, default to the planet file
+  if (_.isEmpty(config.imports.openstreetmap.download)) {
+    sources = [
+      'https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf'
+    ];
+  }
+  else {
+    sources = _.map(config.imports.openstreetmap.download, (source) => source.sourceURL);
+  }
+
+  logger.info(`Downloading sources: ${sources}`);
+
+  fs.mkdir(config.imports.openstreetmap.datapath, { recursive: true }, (err) => {
+    if (err) {
+      logger.error(`error making directory ${config.imports.openstreetmap.datapath}`, err);
+      return callback(err);
+    }
+
+    async.forEach(
+      sources,
+      (source, next) => {
+        downloadSource(config, source, next);
+      },
+      callback
+    );
+  });
+}
+
+function downloadSource(config, sourceUrl, callback) {
+  const targetDir = config.imports.openstreetmap.datapath;
+
+  logger.debug(`downloading ${sourceUrl} to ${targetDir}`);
+  child_process.exec(`cd ${targetDir} && { curl -L -X GET --silent --fail --remote-name ${sourceUrl}; }`, callback);
+}
+
+module.exports = download;

package/util/sendPassthroughStream.js

@@ -0,0 +1,8 @@
+var through = require('through2');
+
+function sendPassthroughStream() {
+  return through.obj(function (doc, enc, next) {
+    next(null, doc);
+  });
+}
+module.exports = sendPassthroughStream;