pelias-openstreetmap 9.1.0 → 9.2.0

package/README.md

@@ -94,6 +94,27 @@ The OSM importer will look for a file with a name matching this value in the con
 
 If downloading from a remote URL, the filename must match the value in `sourceURL`.
 
+A file ending in `.ndjson`, `.ldjson` or `.jsonl`, optionally `.gz` compressed,
+is read as newline delimited JSON instead of being parsed as a pbf. It is
+assumed to be the stored output of a previous `pbf2json` run over the same
+extract, and importing it produces exactly the same documents.
+
+```javascript
+{
+  "imports": {
+    "openstreetmap": {
+      "datapath": "/data/openstreetmap",
+      "import": [{
+        "filename": "pbf2json.pelias.jsonl.gz"
+      }]
+    }
+  }
+}
+```
+
+The two forms can be mixed in one `import` array, and both work with
+`parallelism`. Compressed files are decompressed as they are read.
+
 #### `imports.openstreetmap.layers`
 
 This is an object you can use to define your own layers based on OSM tags. For example, if you wanted a layer `coffee` for just coffee shops you could set:
@@ -136,6 +157,16 @@ discarded. In practice, this affects records with tags such as
 
 By default, or if set to any other value besides `true`, these records will be imported.
 
+#### `imports.openstreetmap.parallelism`
+
+The number of import pipelines to run in parallel. Defaults to `1`, which runs
+the importer as a single process.
+
+Reading a PBF file cannot be parallelized, but everything after the parser can
+be. When set above `1`, a single `pbf2json` reader is started and its output is
+distributed across that many worker processes, each running the full import
+pipeline.
+
 ### Administrative Hierarchy Lookup
 
 OSM records often do not contain information about which city, state (or

package/index.js

@@ -9,4 +9,12 @@ if (_.has(peliasConfig, 'imports.openstreetmap.adminLookup')) {
 
 const importPipeline = require('./stream/importPipeline');
 
-importPipeline.import();
+// a parallelism greater than one runs a single pbf2json reader which fans its
+// output out to that many worker processes, each running the full pipeline
+const parallelism = _.get(peliasConfig, 'imports.openstreetmap.parallelism', 1);
+
+if (parallelism > 1) {
+  require('./parallel/dispatcher').run(parallelism);
+} else {
+  importPipeline.import();
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pelias-openstreetmap",
-  "version": "9.1.0",
+  "version": "9.2.0",
   "engines": {
     "node": ">=22.0.0"
   },

package/parallel/dispatcher.js

@@ -0,0 +1,284 @@
+'use strict';
+
+/**
+  Parallel import dispatcher.
+
+  Reading a pbf file cannot be parallelized, but everything downstream of the
+  parser is stateless per record, so a single pbf2json reader can feed any
+  number of import pipelines. This module runs that reader and forwards its
+  output, split on line boundaries, to N worker processes.
+
+  Blocks are handed to whichever worker is currently accepting writes, so a
+  worker blocked on Elasticsearch does not stall the reader. When every worker
+  is busy the reader is paused, which propagates backpressure to pbf2json.
+
+  Only used when imports.openstreetmap.parallelism is greater than 1.
+**/
+
+const child = require('child_process');
+const path = require('path');
+const os = require('os');
+const fs = require('fs');
+const logger = require('pelias-logger').get('openstreetmap');
+const settings = require('pelias-config').generate(require('../schema'));
+const pbf = require('../stream/pbf');
+const source = require('../stream/source');
+const generateParams = require('pbf2json/lib/generateParams');
+
+const NEWLINE = 0x0a;
+const WORKER_PATH = path.join(__dirname, 'worker.js');
+const PBF2JSON_BIN = path.join(
+  path.dirname(require.resolve('pbf2json')),
+  'build',
+  `pbf2json.${os.platform()}-${os.arch()}`
+);
+
+function run(parallelism) {
+  const configs = sourceConfigs();
+  const state = { shuttingDown: false, exitCode: 0, workers: [], reader: null };
+
+  state.workers = startWorkers(parallelism, state);
+  const dispatcher = createDispatcher(state.workers);
+
+  ['SIGINT', 'SIGTERM'].forEach((signal) => {
+    process.on(signal, () => {
+      state.shuttingDown = true;
+      killChildren(state);
+      process.exit(1);
+    });
+  });
+
+  readSequentially(configs, dispatcher, state, () => {
+    logger.info('pbf parsing complete, waiting for workers to flush');
+    state.shuttingDown = true;
+    dispatcher.end();
+  });
+}
+
+// one reader config per configured import file. stored pbf2json output needs
+// neither the pbf2json options nor a leveldb cache
+function sourceConfigs() {
+  const osm = settings.imports.openstreetmap;
+
+  return osm.import.map((entry) => {
+    const file = path.join(osm.datapath, entry.filename);
+
+    const conf = source.isJsonSource(file) ?
+      { file: file, importVenues: entry.importVenues } :
+      pbf.config({
+        file: file,
+        leveldb: osm.leveldbpath,
+        importVenues: entry.importVenues
+      });
+
+    [conf.file, conf.leveldb].filter(Boolean).forEach((target) => {
+      try {
+        fs.statSync(target);
+      } catch (e) {
+        throw new Error('failed to stat path: ' + target);
+      }
+    });
+
+    return conf;
+  });
+}
+
+function startWorkers(parallelism, state, workerPath) {
+  const workers = [];
+
+  for (let id = 0; id < parallelism; id++) {
+    workers.push(spawnWorker(id, workers, state, workerPath || WORKER_PATH));
+  }
+
+  logger.info(`started ${parallelism} import workers`);
+  return workers;
+}
+
+function spawnWorker(id, workers, state, workerPath) {
+  const proc = child.spawn(process.execPath, process.execArgv.concat(workerPath), {
+    stdio: ['pipe', 'inherit', 'inherit'],
+    env: Object.assign({}, process.env, { PELIAS_OSM_WORKER_ID: String(id) })
+  });
+
+  const worker = { id: id, proc: proc, ready: true, alive: true };
+
+  // a closed stdin surfaces as the exit below, no need to also throw here
+  proc.stdin.on('error', () => {});
+
+  proc.on('exit', (code, signal) => {
+    worker.alive = false;
+    worker.ready = false;
+
+    if (!state.shuttingDown) {
+      fatal(state, `worker ${id} exited early (code ${code}, signal ${signal})`);
+      return;
+    }
+
+    if (code !== 0) {
+      logger.error(`worker ${id} exited with code ${code}, signal ${signal}`);
+      state.exitCode = 1;
+    }
+
+    if (workers.every((w) => !w.alive)) {
+      logger.info('import complete');
+      process.exit(state.exitCode);
+    }
+  });
+
+  return worker;
+}
+
+function createDispatcher(workers) {
+  let cursor = 0;
+  let leftover = null;
+  let source = null;
+  let paused = false;
+
+  // round robin over the workers currently accepting writes
+  function claim() {
+    let fallback = null;
+
+    for (let i = 0; i < workers.length; i++) {
+      const worker = workers[(cursor + i) % workers.length];
+      if (!worker.alive) { continue; }
+      if (worker.ready) {
+        cursor = (cursor + i + 1) % workers.length;
+        return worker;
+      }
+      fallback = fallback || worker;
+    }
+
+    // every worker is busy, buffer into one of them rather than dropping the
+    // block: the reader is paused immediately after this write
+    return fallback;
+  }
+
+  function resume() {
+    if (paused && source) {
+      paused = false;
+      source.resume();
+    }
+  }
+
+  function send(block) {
+    const worker = claim();
+    if (!worker) { return; }
+
+    const wasReady = worker.ready;
+
+    if (worker.proc.stdin.write(block)) {
+      worker.ready = true;
+    } else if (wasReady) {
+      worker.ready = false;
+      worker.proc.stdin.once('drain', () => {
+        worker.ready = true;
+        resume();
+      });
+    }
+  }
+
+  function onData(chunk) {
+    const buf = leftover ? Buffer.concat([leftover, chunk]) : chunk;
+    const idx = buf.lastIndexOf(NEWLINE);
+
+    // no complete line yet, keep accumulating
+    if (idx === -1) {
+      leftover = buf;
+      return;
+    }
+
+    leftover = idx + 1 < buf.length ? Buffer.from(buf.subarray(idx + 1)) : null;
+    send(buf.subarray(0, idx + 1));
+
+    if (source && !workers.some((worker) => worker.ready)) {
+      paused = true;
+      source.pause();
+    }
+  }
+
+  return {
+    attach: (stdout) => {
+      source = stdout;
+      paused = false;
+      stdout.on('data', onData);
+    },
+    end: () => {
+      if (leftover && leftover.length) {
+        send(Buffer.concat([leftover, Buffer.from('\n')]));
+        leftover = null;
+      }
+      workers.forEach((worker) => {
+        if (worker.alive) { worker.proc.stdin.end(); }
+      });
+    }
+  };
+}
+
+function readSequentially(configs, dispatcher, state, done) {
+  let index = 0;
+
+  (function next() {
+    if (index >= configs.length) { return done(); }
+
+    const conf = configs[index++];
+    logger.info('Creating read stream for: ' + conf.file);
+
+    if (source.isJsonSource(conf.file)) {
+      const stream = source.createByteStream(conf.file);
+
+      stream.on('error', (err) => fatal(state, 'failed to read ' + conf.file + ': ' + err.message));
+      stream.on('end', next);
+
+      state.reader = stream;
+      dispatcher.attach(stream);
+      return;
+    }
+
+    const proc = child.spawn(PBF2JSON_BIN, generateParams(conf), {
+      stdio: ['ignore', 'pipe', 'pipe']
+    });
+
+    proc.stderr.on('data', (data) => {
+      data.toString('utf8').trim().split('\n').forEach((line) => {
+        if (line.indexOf('[info]') === -1 && line.indexOf('[warn]') === -1) {
+          logger.error('[pbf2json]: ' + line);
+        }
+      });
+    });
+
+    proc.on('close', (code) => {
+      if (code !== 0) {
+        return fatal(state, 'pbf2json exited with code ' + code);
+      }
+      next();
+    });
+
+    state.reader = proc;
+    dispatcher.attach(proc.stdout);
+  })();
+}
+
+function killChildren(state) {
+  if (state.reader) {
+    if (typeof state.reader.kill === 'function') { state.reader.kill(); }
+    else { state.reader.destroy(); }
+  }
+  state.workers.forEach((worker) => {
+    if (worker.alive) { worker.proc.kill(); }
+  });
+}
+
+function fatal(state, message) {
+  logger.error(message);
+  state.shuttingDown = true;
+  killChildren(state);
+  process.exit(1);
+}
+
+module.exports.run = run;
+
+// exported for testing
+module.exports.createDispatcher = createDispatcher;
+module.exports.startWorkers = startWorkers;
+module.exports.readSequentially = readSequentially;
+module.exports.sourceConfigs = sourceConfigs;

package/parallel/ndjson.js

@@ -0,0 +1,58 @@
+'use strict';
+
+/**
+  Decodes a byte stream of newline delimited JSON into an object stream.
+  Used by the parallel import workers to read the pbf2json output forwarded
+  to them by the dispatcher.
+
+  A line which fails to parse is fatal: it means the stream was truncated or
+  the dispatcher mis-aligned a block boundary, and silently dropping records
+  would produce an incomplete index.
+**/
+
+const { Transform } = require('stream');
+const NEWLINE = 0x0a;
+
+class NDJSONDecoder extends Transform {
+  constructor() {
+    super({ writableObjectMode: false, readableObjectMode: true });
+    this.leftover = null;
+  }
+
+  _transform(chunk, enc, next) {
+    const buf = this.leftover ? Buffer.concat([this.leftover, chunk]) : chunk;
+    let start = 0;
+    let idx = buf.indexOf(NEWLINE, start);
+
+    while (idx !== -1) {
+      const err = this.decode(buf.subarray(start, idx));
+      if (err) { return next(err); }
+      start = idx + 1;
+      idx = buf.indexOf(NEWLINE, start);
+    }
+
+    this.leftover = start < buf.length ? buf.subarray(start) : null;
+    next();
+  }
+
+  _flush(next) {
+    const leftover = this.leftover;
+    this.leftover = null;
+    next(leftover ? this.decode(leftover) : null);
+  }
+
+  // returns an Error when the line is not valid JSON, else undefined
+  decode(line) {
+    if (!line.length) { return; }
+    try {
+      const obj = JSON.parse(line);
+      if (obj) { this.push(obj); }
+    } catch (e) {
+      return new Error('failed to decode json: ' + line.toString('utf8').slice(0, 200));
+    }
+  }
+}
+
+module.exports.create = function () {
+  return new NDJSONDecoder();
+};

package/parallel/worker.js

@@ -0,0 +1,23 @@
+'use strict';
+
+/**
+  A single import pipeline reading newline delimited JSON from stdin instead of
+  spawning its own pbf2json reader. Spawned by the dispatcher, never run directly.
+**/
+
+const logger = require('pelias-logger').get('openstreetmap');
+const importPipeline = require('../stream/importPipeline');
+const ndjson = require('./ndjson');
+
+const id = process.env.PELIAS_OSM_WORKER_ID || '0';
+const source = ndjson.create();
+
+logger.info(`import worker ${id} ready`);
+
+source.on('error', (err) => {
+  logger.error(`worker ${id} failed to decode input: ${err.message}`);
+  process.exit(1);
+});
+
+process.stdin.pipe(source);
+importPipeline.import(source, `openstreetmap-${id}`);

package/schema.js

@@ -9,6 +9,7 @@ const Joi = require('joi');
 // importVenues: boolean (optional)
 // download: array of objects containing sourceURL (optional)
 // deduplicate: boolean (optional)
+// parallelism: integer >= 1 (optional, defaults to 1)
 module.exports = Joi.object().keys({
   imports: Joi.object().keys({
     openstreetmap: Joi.object().keys({
@@ -22,6 +23,7 @@ module.exports = Joi.object().keys({
         sourceURL: Joi.string()
       }).requiredKeys('sourceURL').unknown(true)),
       deduplicate: Joi.boolean(),
+      parallelism: Joi.number().integer().min(1).default(1),
       addressTags: Joi.array().items(Joi.string()),
       layers: Joi.object().pattern(
         Joi.string(),

package/stream/importPipeline.js

@@ -21,8 +21,11 @@ streams.dbMapper = require('pelias-model').createDocumentMapperStream;
 streams.elasticsearch = require('pelias-dbclient');
 
 // default import pipeline
-streams.import = function(){
-  streams.pbfParser()
+// source and name are only supplied by the parallel dispatcher's workers, which
+// read pbf2json output from stdin rather than parsing a pbf themselves, and name
+// themselves individually so dbclient stats and output files stay separable
+streams.import = function(source, name){
+  ( source || streams.pbfParser() )
     .pipe( streams.docConstructor() )
     .pipe( streams.addressesWithoutStreet() )
     .pipe( streams.tagMapper() )
@@ -34,7 +37,7 @@ streams.import = function(){
     .pipe( streams.popularityMapper() )
     .pipe( streams.adminLookup() )
     .pipe( streams.dbMapper() )
-    .pipe( streams.elasticsearch({name: 'openstreetmap'}) );
+    .pipe( streams.elasticsearch({name: name || 'openstreetmap'}) );
 };
 
 module.exports = streams;

package/stream/multiple_pbfs.js

@@ -1,5 +1,5 @@
 var combinedStream = require('combined-stream');
-var pbf = require('./pbf');
+var source = require('./source');
 var path = require('path');
 var logger = require('pelias-logger').get('openstreetmap');
 
@@ -15,7 +15,7 @@ function createCombinedStream(){
     };
     fullStream.append(function(next){
       logger.info('Creating read stream for: ' + conf.file);
-      next(pbf.parser(conf));
+      next(source.createRecordStream(conf));
     });
   });
 

package/stream/source.js

@@ -0,0 +1,93 @@
+'use strict';
+
+/**
+  Chooses how to read a configured OSM source file.
+
+  A .pbf is parsed by pbf2json, which first builds a leveldb cache of node
+  positions. That cache is the slowest part of an OSM import - hours for a
+  planet file - and its output is deterministic, so it is often produced once
+  and stored.
+
+  A .ndjson, .ldjson or .jsonl file is assumed to be exactly that: the newline
+  delimited output of a previous pbf2json run. It is read directly, skipping
+  both the parse and the cache build. Since that output is usually kept
+  compressed, a .gz suffix is decompressed on the way through.
+**/
+
+const fs = require('fs');
+const path = require('path');
+const zlib = require('zlib');
+const { pipeline } = require('stream');
+const pbf = require('./pbf');
+const ndjson = require('../parallel/ndjson');
+const logger = require('pelias-logger').get('openstreetmap');
+
+const JSON_EXTENSIONS = ['.ndjson', '.ldjson', '.jsonl'];
+const GZIP_EXTENSION = '.gz';
+
+function isCompressed(file) {
+  return path.extname(file || '').toLowerCase() === GZIP_EXTENSION;
+}
+
+// the name without its compression suffix, so 'planet.jsonl.gz' is recognised
+// by the same rule as 'planet.jsonl'
+function uncompressedName(file) {
+  return isCompressed(file) ? (file || '').slice(0, -GZIP_EXTENSION.length) : (file || '');
+}
+
+/**
+ * True when this file holds stored pbf2json output rather than a pbf.
+ */
+function isJsonSource(file) {
+  return JSON_EXTENSIONS.includes(path.extname(uncompressedName(file)).toLowerCase());
+}
+
+// options which pbf2json applies while parsing, and which a stored file has
+// already been produced with
+function warnIgnoredOptions(conf) {
+  if (conf.importVenues === false) {
+    logger.warn(
+      `importVenues is ignored for ${path.basename(conf.file)}: which tags were ` +
+      'extracted was decided when the file was generated'
+    );
+  }
+}
+
+/**
+ * A stream of OSM record objects, from either source type.
+ */
+function createRecordStream(conf) {
+  if (!isJsonSource(conf.file)) {
+    return pbf.parser(conf);
+  }
+
+  warnIgnoredOptions(conf);
+
+  // pipeline rather than pipe: a read or decompression failure has to surface
+  // on the stream the pipeline is consuming, not be lost upstream
+  return pipeline(createByteStream(conf.file), ndjson.create(), () => {});
+}
+
+/**
+ * The raw newline delimited bytes of a stored file, for the parallel
+ * dispatcher: it splits on newlines itself and never parses the json.
+ *
+ * Decompression, when needed, happens in this process. For a planet sized file
+ * that is a few percent of one core spread over the whole import, far less than
+ * the pbf parse it replaces.
+ */
+function createByteStream(file) {
+  const raw = fs.createReadStream(file);
+  if (!isCompressed(file)) { return raw; }
+
+  // pipeline so the file handle is closed if decompression fails, or if the
+  // reader is destroyed when the import is interrupted
+  return pipeline(raw, zlib.createGunzip(), () => {});
+}
+
+module.exports = {
+  isJsonSource: isJsonSource,
+  isCompressed: isCompressed,
+  createRecordStream: createRecordStream,
+  createByteStream: createByteStream
+};