pelias-openstreetmap 5.13.0

package/.dockerignore

@@ -0,0 +1,2 @@
+.git
+node_modules

package/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*]
+charset = utf-8
+insert_final_newline = true
+
+[*.{js,yml,json}]
+indent_style = space
+indent_size = 2

package/.github/workflows/push.yml

@@ -0,0 +1,55 @@
+name: Continuous Integration
+on: push
+jobs:
+  unit-tests:
+    runs-on: '${{ matrix.os }}'
+    strategy:
+      matrix:
+        os:
+          - ubuntu-20.04
+        node-version:
+          - 12.x
+          - 14.x
+          - 16.x
+    steps:
+      - uses: actions/checkout@v2
+      - name: 'Install node.js ${{ matrix.node-version }}'
+        uses: actions/setup-node@v2-beta
+        with:
+          node-version: '${{ matrix.node-version }}'
+      - name: Run unit tests
+        run: |
+          npm install
+          npm run ci
+  npm-publish:
+    needs: unit-tests
+    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Node.js
+        uses: actions/setup-node@v2-beta
+        with:
+          node-version: 16.x
+      - name: Run semantic-release
+        env:
+          GH_TOKEN: ${{ secrets.GH_SEMANTIC_RELEASE_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+        run: >
+          if [[ -n "$GH_TOKEN" && -n "$NPM_TOKEN" ]]; then
+            curl "https://raw.githubusercontent.com/pelias/ci-tools/master/semantic-release.sh" | bash -
+          fi
+  build-docker-images:
+    # run this job if the unit tests passed and the npm-publish job was a success or was skipped
+    # note: github actions won't run a job if you don't call one of the status check functions, so `always()` is called since it evalutes to `true`
+    if: ${{ always() && needs.unit-tests.result == 'success' && (needs.npm-publish.result == 'success' || needs.npm-publish.result == 'skipped') }}
+    needs: [unit-tests, npm-publish]
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build Docker images
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: |
+          curl "https://raw.githubusercontent.com/pelias/ci-tools/master/build-docker-images.sh" | bash -

package/.jshintignore

@@ -0,0 +1,6 @@
+/node_modules/
+esclient.log
+*.log
+nohup.out
+test/*.pbf
+coverage

package/.jshintrc

@@ -0,0 +1,22 @@
+{
+  "esversion": 6,
+  "node": true,
+  "curly": true,
+  "eqeqeq": true,
+  "freeze": true,
+  "immed": true,
+  "indent": 2,
+  "latedef": false,
+  "newcap": true,
+  "noarg": true,
+  "noempty": true,
+  "nonbsp": true,
+  "nonew": true,
+  "plusplus": false,
+  "quotmark": "single",
+  "undef": true,
+  "unused": false,
+  "maxparams": 4,
+  "maxdepth": 5,
+  "maxlen": 140
+}

package/Dockerfile

@@ -0,0 +1,23 @@
+# base image
+FROM pelias/baseimage
+
+# downloader apt dependencies
+# note: this is done in one command in order to keep down the size of intermediate containers
+RUN apt-get update && apt-get install -y bzip2 unzip && rm -rf /var/lib/apt/lists/*
+
+# change working dir
+ENV WORKDIR /code/pelias/openstreetmap
+WORKDIR ${WORKDIR}
+
+# copy package.json first to prevent npm install being rerun when only code changes
+COPY ./package.json ${WORKDIR}
+RUN npm install
+
+# add local code
+ADD . ${WORKDIR}
+
+# run tests, clean up LevelDB lockfile
+RUN npm test && rm -rf /tmp/*
+
+# run as the pelias user
+USER pelias

package/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Mapzen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,184 @@
+<p align="center">
+  <img height="100" src="https://raw.githubusercontent.com/pelias/design/master/logo/pelias_github/Github_markdown_hero.png">
+</p>
+<h3 align="center">A modular, open-source search engine for our world.</h3>
+<p align="center">Pelias is a geocoder powered completely by open data, available freely to everyone.</p>
+<p align="center">
+<a href="https://en.wikipedia.org/wiki/MIT_License"><img src="https://img.shields.io/github/license/pelias/api?style=flat&color=orange" /></a>
+<a href="https://hub.docker.com/u/pelias"><img src="https://img.shields.io/docker/pulls/pelias/api?style=flat&color=informational" /></a>
+<a href="https://gitter.im/pelias/pelias"><img src="https://img.shields.io/gitter/room/pelias/pelias?style=flat&color=yellow" /></a>
+</p>
+<p align="center">
+	<a href="https://github.com/pelias/docker">Local Installation</a> ·
+        <a href="https://geocode.earth">Cloud Webservice</a> ·
+	<a href="https://github.com/pelias/documentation">Documentation</a> ·
+	<a href="https://gitter.im/pelias/pelias">Community Chat</a>
+</p>
+<details open>
+<summary>What is Pelias?</summary>
+<br />
+Pelias is a search engine for places worldwide, powered by open data. It turns addresses and place names into geographic coordinates, and turns geographic coordinates into places and addresses. With Pelias, you’re able to turn your users’ place searches into actionable geodata and transform your geodata into real places.
+<br /><br />
+We think open data, open source, and open strategy win over proprietary solutions at any part of the stack and we want to ensure the services we offer are in line with that vision. We believe that an open geocoder improves over the long-term only if the community can incorporate truly representative local knowledge.
+</details>
+
+# Pelias OpenStreetMap importer
+
+## Overview
+
+The OpenStreetMap importer handles importing data from [OpenStreetMap](https://www.openstreetmap.org/) into Elasticsearch for use by Pelias.
+
+It includes logic for filtering to select only data relevant for geocoding, transforming it to match the Pelias data model, and augmenting the data as required.
+
+## Prerequisites
+
+See [Pelias software requirements](https://github.com/pelias/documentation/blob/master/requirements.md)
+
+## Clone and Install dependencies
+
+> For instructions on setting up Pelias as a whole, see our [getting started guide](https://github.com/pelias/documentation/blob/master/getting_started_install.md). Further instructions here pertain to the OSM importer only
+
+```bash
+$ git clone https://github.com/pelias/openstreetmap.git && cd openstreetmap;
+$ npm install
+```
+
+## Download data
+
+The importer will accept any valid `pbf` extract you have, such as a full planet file (50GB+) from [planet.openstreetmap.org](https://planet.openstreetmap.org) or [download.geofabrik.de](https://download.geofabrik.de)
+You can use the included download script to obtain the desired `pbf` files as follows. In the configuration file you can
+specify which files are to be downloaded. They will all be downloaded to the `imports.openstreetmap.datapath` directory.
+
+If no download sources are specified in the configuration, the entire planet file will be downloaded. Keep in mind this file is quite large.
+
+```bash
+$ PELIAS_CONFIG=<path-to-config> npm run download
+```
+
+## Configuration
+
+In order to tell the importer the location of your downloads, temp space and environmental settings you will first need to create a `~/pelias.json` file.
+
+See [the config](https://github.com/pelias/config) documentation for details on the structure of this file. Your relevant config info for the openstreetmap module might look something like this:
+
+```javascript
+{
+  "imports": {
+    "openstreetmap": {
+      "download": [{
+        "sourceURL": "https://s3.amazonaws.com/metro-extracts.nextzen.org/portland_oregon.osm.pbf"
+      }],
+      "datapath": "/mnt/pelias/openstreetmap",
+      "leveldbpath": "/tmp",
+      "import": [{
+        "filename": "portland_oregon.osm.pbf"
+      }]
+    }
+  }
+}
+```
+
+### Configuration Settings
+
+#### `imports.openstreetmap.datapath`
+
+This is the directory where the OSM importer will look for files to import. If configured it will also download files to this location.
+
+#### `imports.openstreetmap.download[0].sourceURL`
+
+A URL to download when the download script (in `./bin/download`) is run. Will be downloaded to the `datapath` directory.
+
+#### `imports.openstreetmap.import[0].filename`
+
+The OSM importer will look for a file with a name matching this value in the configured `datapath` directory when importing data.
+
+If downloading from a remote URL, the filename must match the value in `sourceURL`.
+
+#### `imports.openstreetmap.leveldbpath`
+
+This is the directory where temporary files will be stored in order to
+denormalize OSM ways and relations. In the case of a planet import it is best
+to have at least 100GB free.
+
+Defaults to `tmp`.
+
+#### `imports.openstreetmap.importVenues`
+
+By default, the OSM importer imports both venue records and addresses. If set to false, only address records will be imported.
+
+#### `imports.openstreetmap.removeDisusedVenues`
+
+If set to boolean `true`, `venue`s with popularity below `0` (as determined by OSM tags) will be
+discarded. In practice, this affects records with tags such as
+[`disused`](https://wiki.openstreetmap.org/wiki/Key:disused#disused:_namespace),
+[`amenity:disused`](https://wiki.openstreetmap.org/wiki/Tag:amenity%3Ddisused) and
+[`abandoned`](https://wiki.openstreetmap.org/wiki/Key:abandoned:)
+
+By default, or if set to any other value besides `true`, these records will be imported.
+
+### Administrative Hierarchy Lookup
+
+OSM records often do not contain information about which city, state (or
+other region like province), or country that they belong to. Pelias has the
+ability to compute these values from [Who's on First](http://www.whosonfirst.org/) data.
+For more info on how admin lookup works, see the documentation for
+[pelias/wof-admin-lookup](https://github.com/pelias/wof-admin-lookup). By default,
+adminLookup is enabled.  To disable, set `imports.adminLookup.enabled` to `false` in Pelias config.
+
+**Note:** Admin lookup requires loading around 5GB of data into memory.
+
+## Running an import
+
+This will start the import process. It may take a few minutes to load administrative data and begin processing the OSM PBF file, then you should see regular progress updates in the terminal.
+
+```bash
+$ npm start
+```
+
+## How long does it take?
+
+If all goes well, you should see between 6000-7000 records imported per second on a modern machine. A full planet install will import about 80 million records, whereas most city extracts will import at most a few thousand.
+
+These counts are of records containing valid location names to search on, data which is not directly searchable by the end user, such as fire hydrants, lamp posts, etc is not imported.
+
+If you are looking to run a planet-wide cluster like the one we provide for [geocode.earth](https://geocode.earth) please see our documentation on [full planet builds](https://github.com/pelias/documentation/blob/master/full_planet_considerations.md#importer-machine).
+
+## Issues
+
+If you have any issues getting set up or the documentation is missing something, please open an issue here: https://github.com/pelias/openstreetmap/issues
+
+## Contributing
+
+Please fork and pull request against upstream master on a feature branch.
+
+Pretty please; provide unit tests and script fixtures in the `test` directory.
+
+## Code Linting
+
+A `.jshintrc` file is provided which contains a linting config, usually your text editor will understand this config and give you inline hints on code style and readability.
+
+These settings are strictly enforced when you do a `git commit`, you can execute `git commit` at any time to run the linter against your code.
+
+### Running Unit Tests
+
+```bash
+$ npm test
+```
+
+### Running End-to-End Tests
+
+These tests run the entire pipeline against a small PBF extract to assert that the individual units work as expected when wired together.
+
+```bash
+$ npm run end-to-end
+```
+
+## Code Coverage
+
+```bash
+$ npm run coverage
+```
+
+### Continuous Integration
+
+CI tests every change against all supported Node.js versions.

package/bin/download

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec node util/download_data.js

package/bin/start

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec node --max_old_space_size=8000 index.js

package/bin/test

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# run tests with pipefail to avoid false passes
+# see https://github.com/pelias/pelias/issues/744
+set -o pipefail
+
+NODE_ENV=test node test/run.js | npx tap-spec