osmforge 1.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- osmforge-1.1.2/LICENSE +21 -0
- osmforge-1.1.2/PKG-INFO +374 -0
- osmforge-1.1.2/README.md +348 -0
- osmforge-1.1.2/osmforge/__init__.py +4 -0
- osmforge-1.1.2/osmforge/client.py +154 -0
- osmforge-1.1.2/osmforge/docker-compose.yaml +32 -0
- osmforge-1.1.2/osmforge/download.py +114 -0
- osmforge-1.1.2/pyproject.toml +84 -0
osmforge-1.1.2/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tom Freeman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
osmforge-1.1.2/PKG-INFO
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: osmforge
|
|
3
|
+
Version: 1.1.2
|
|
4
|
+
Summary: Self-hosted OpenStreetMap stack and Python client for spatial analysis
|
|
5
|
+
Home-page: https://github.com/Tom3man/osm-forge
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: osm,openstreetmap,gis,geospatial,postgis,propagation
|
|
8
|
+
Author: Tom Freeman
|
|
9
|
+
Author-email: tomrfreeman3@gmail.com
|
|
10
|
+
Requires-Python: >=3.12,<4.0
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
18
|
+
Requires-Dist: geopandas (>=1.1.1,<2.0.0)
|
|
19
|
+
Requires-Dist: platformdirs (>=4.0,<5.0)
|
|
20
|
+
Requires-Dist: requests (>=2.32.5,<3.0.0)
|
|
21
|
+
Requires-Dist: shapely (>=2.1.2,<3.0.0)
|
|
22
|
+
Project-URL: Documentation, https://tom3man.github.io/osm-forge/
|
|
23
|
+
Project-URL: Repository, https://github.com/Tom3man/osm-forge
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# OSMForge
|
|
27
|
+
|
|
28
|
+
A self-hosted OpenStreetMap stack that downloads regional OSM extracts, ingests
|
|
29
|
+
them into a local PostGIS database, and serves a GeoJSON API optimised for
|
|
30
|
+
spatial analysis — particularly **radio-frequency propagation modelling**.
|
|
31
|
+
|
|
32
|
+
Public OSM APIs (Overpass, Nominatim) have strict rate limits and aren't suited
|
|
33
|
+
to bulk or repeated spatial queries. OSMForge gives you unlimited local access
|
|
34
|
+
to the same data.
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## Contents
|
|
39
|
+
|
|
40
|
+
- [Architecture](#architecture)
|
|
41
|
+
- [Prerequisites](#prerequisites)
|
|
42
|
+
- [Quick start](#quick-start)
|
|
43
|
+
- [Adding regions](#adding-regions)
|
|
44
|
+
- [Makefile reference](#makefile-reference)
|
|
45
|
+
- [API reference](#api-reference)
|
|
46
|
+
- [Python client](#python-client)
|
|
47
|
+
- [Data layers](#data-layers)
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## Architecture
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
Geofabrik ──download──▶ *.osm.pbf
|
|
55
|
+
│
|
|
56
|
+
osm2pgsql (Lua flex)
|
|
57
|
+
│
|
|
58
|
+
osm schema (raw)
|
|
59
|
+
osm_points / osm_lines / osm_polygons
|
|
60
|
+
│
|
|
61
|
+
make layers (SQL)
|
|
62
|
+
│
|
|
63
|
+
app schema (classified)
|
|
64
|
+
osm_buildings / osm_roads / osm_water /
|
|
65
|
+
osm_vegetation / osm_landuse /
|
|
66
|
+
osm_structures / osm_terrain
|
|
67
|
+
│
|
|
68
|
+
FastAPI :8000
|
|
69
|
+
│
|
|
70
|
+
osmforge.client
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
The `osm` schema holds raw data exactly as loaded by osm2pgsql.
|
|
74
|
+
The `app` schema holds pre-classified, indexed tables ready for querying.
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## Prerequisites
|
|
79
|
+
|
|
80
|
+
| Tool | Version | Notes |
|
|
81
|
+
|---|---|---|
|
|
82
|
+
| Docker + Compose v2 | latest | `docker compose version` |
|
|
83
|
+
| osm2pgsql | ≥ 1.9 | `osm2pgsql --version` |
|
|
84
|
+
| Python | ≥ 3.12 | |
|
|
85
|
+
| Poetry | ≥ 1.8 | `pip install poetry` |
|
|
86
|
+
|
|
87
|
+
Install Python dependencies:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
poetry install
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## Quick start
|
|
96
|
+
|
|
97
|
+
### 1 — Start the database
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
make up
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
This starts a PostGIS 17 container on `localhost:5432` and the FastAPI container
|
|
104
|
+
on `localhost:8000`. On first run Docker will pull the images.
|
|
105
|
+
|
|
106
|
+
### 2 — Download an OSM extract
|
|
107
|
+
|
|
108
|
+
Extracts come from [Geofabrik](https://download.geofabrik.de). Pass the full
|
|
109
|
+
path from the Geofabrik URL:
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
make download REGION=europe/united-kingdom/england/isle-of-wight
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
The file lands in `osmforge/data/` and is skipped on subsequent runs unless you
|
|
116
|
+
pass `FORCE=--force`.
|
|
117
|
+
|
|
118
|
+
### 3 — Ingest + build layers
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
make ingest # load every *.osm.pbf in osmforge/data/ into osm.*
|
|
122
|
+
make layers # build classified app.* tables
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Or in one step:
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
make rebuild
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### 4 — Query the API
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
curl "http://localhost:8000/propagation/bbox?min_lon=-1.35&min_lat=50.65&max_lon=-1.1&max_lat=50.78"
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
## Adding regions
|
|
140
|
+
|
|
141
|
+
Download as many regions as you like before ingesting — `make ingest` loads
|
|
142
|
+
everything in `osmforge/data/` in a single pass.
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
# individual counties
|
|
146
|
+
make download REGION=europe/united-kingdom/england/west-midlands
|
|
147
|
+
make download REGION=europe/united-kingdom/england/staffordshire
|
|
148
|
+
make download REGION=europe/united-kingdom/england/warwickshire
|
|
149
|
+
|
|
150
|
+
# then ingest them all together
|
|
151
|
+
make rebuild
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
To see what Geofabrik has available, browse:
|
|
155
|
+
- England: https://download.geofabrik.de/europe/united-kingdom/england.html
|
|
156
|
+
- Full index: https://download.geofabrik.de
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## Makefile reference
|
|
161
|
+
|
|
162
|
+
| Command | Description |
|
|
163
|
+
|---|---|
|
|
164
|
+
| `make up` | Start PostGIS + API containers |
|
|
165
|
+
| `make down` | Stop containers (data volume preserved) |
|
|
166
|
+
| `make download REGION=<path>` | Download a PBF from Geofabrik |
|
|
167
|
+
| `make ingest` | Load all PBFs in `osmforge/data/` via osm2pgsql |
|
|
168
|
+
| `make layers` | Build `app.*` tables from raw `osm.*` data |
|
|
169
|
+
| `make rebuild` | `ingest` + `layers` in one step |
|
|
170
|
+
| `make rebuild-api` | Rebuild the API Docker image after code changes |
|
|
171
|
+
| `make clean` | Destroy containers **and** data volume (full reset) |
|
|
172
|
+
| `make interface` | Open a psql shell to the database |
|
|
173
|
+
| `make help` | List all targets |
|
|
174
|
+
|
|
175
|
+
---
|
|
176
|
+
|
|
177
|
+
## API reference
|
|
178
|
+
|
|
179
|
+
The API runs at `http://localhost:8000`. Interactive docs at
|
|
180
|
+
`http://localhost:8000/docs`.
|
|
181
|
+
|
|
182
|
+
---
|
|
183
|
+
|
|
184
|
+
### `GET /health`
|
|
185
|
+
|
|
186
|
+
```
|
|
187
|
+
200 {"status": "ok"}
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
---
|
|
191
|
+
|
|
192
|
+
### `GET /propagation/bbox`
|
|
193
|
+
|
|
194
|
+
Return classified OSM features within a bounding box.
|
|
195
|
+
|
|
196
|
+
**Query parameters**
|
|
197
|
+
|
|
198
|
+
| Parameter | Type | Required | Description |
|
|
199
|
+
|---|---|---|---|
|
|
200
|
+
| `min_lon` | float | ✓ | West edge (WGS-84) |
|
|
201
|
+
| `min_lat` | float | ✓ | South edge |
|
|
202
|
+
| `max_lon` | float | ✓ | East edge |
|
|
203
|
+
| `max_lat` | float | ✓ | North edge |
|
|
204
|
+
| `layers` | string (repeat) | | Subset of layers (default: all) |
|
|
205
|
+
| `limit` | int | | Max features returned |
|
|
206
|
+
|
|
207
|
+
**Example**
|
|
208
|
+
|
|
209
|
+
```
|
|
210
|
+
GET /propagation/bbox?min_lon=-1.35&min_lat=50.65&max_lon=-1.1&max_lat=50.78&layers=buildings&layers=terrain
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
**Response** — GeoJSON FeatureCollection. Each feature's `properties`:
|
|
214
|
+
|
|
215
|
+
```json
|
|
216
|
+
{
|
|
217
|
+
"osm_id": 123456,
|
|
218
|
+
"name": "St Mary's Church",
|
|
219
|
+
"layer": "buildings",
|
|
220
|
+
"feature_class": "church",
|
|
221
|
+
"height_m": 18.0,
|
|
222
|
+
"levels": "3",
|
|
223
|
+
"material": "stone"
|
|
224
|
+
}
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
229
|
+
### `POST /propagation/geometry`
|
|
230
|
+
|
|
231
|
+
Return classified OSM features intersecting an arbitrary polygon or
|
|
232
|
+
multipolygon. Useful when you already have a coverage or study-area polygon.
|
|
233
|
+
|
|
234
|
+
**Request body**
|
|
235
|
+
|
|
236
|
+
```json
|
|
237
|
+
{
|
|
238
|
+
"geometry": {
|
|
239
|
+
"type": "Polygon",
|
|
240
|
+
"coordinates": [[[<lon>, <lat>], ...]]
|
|
241
|
+
},
|
|
242
|
+
"layers": ["buildings", "vegetation", "terrain"],
|
|
243
|
+
"limit": 5000
|
|
244
|
+
}
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
| Field | Type | Required | Description |
|
|
248
|
+
|---|---|---|---|
|
|
249
|
+
| `geometry` | GeoJSON geometry | ✓ | Polygon or MultiPolygon, WGS-84 |
|
|
250
|
+
| `layers` | array of strings | | Subset of layers (default: all) |
|
|
251
|
+
| `limit` | int | | Max features returned |
|
|
252
|
+
|
|
253
|
+
---
|
|
254
|
+
|
|
255
|
+
### `GET /features/bbox`
|
|
256
|
+
|
|
257
|
+
Return **raw** OSM features (all tags) within a bounding box. Useful for
|
|
258
|
+
exploration; use `/propagation/bbox` for modelling workflows.
|
|
259
|
+
|
|
260
|
+
**Query parameters** — same bbox params as above, plus optional `limit`.
|
|
261
|
+
|
|
262
|
+
---
|
|
263
|
+
|
|
264
|
+
## Python client
|
|
265
|
+
|
|
266
|
+
Install the package then import the client:
|
|
267
|
+
|
|
268
|
+
```python
|
|
269
|
+
from osmforge.client import OSMClient
|
|
270
|
+
|
|
271
|
+
client = OSMClient() # default: http://localhost:8000
|
|
272
|
+
# or: OSMClient("http://my-server:8000")
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
All methods return a `geopandas.GeoDataFrame` (CRS EPSG:4326).
|
|
276
|
+
|
|
277
|
+
---
|
|
278
|
+
|
|
279
|
+
### `client.propagation_bbox`
|
|
280
|
+
|
|
281
|
+
```python
|
|
282
|
+
gdf = client.propagation_bbox(
|
|
283
|
+
min_lon=-1.35,
|
|
284
|
+
min_lat=50.65,
|
|
285
|
+
max_lon=-1.1,
|
|
286
|
+
max_lat=50.78,
|
|
287
|
+
)
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
Filter to specific layers:
|
|
291
|
+
|
|
292
|
+
```python
|
|
293
|
+
gdf = client.propagation_bbox(
|
|
294
|
+
min_lon=-1.35, min_lat=50.65,
|
|
295
|
+
max_lon=-1.1, max_lat=50.78,
|
|
296
|
+
layers=["buildings", "terrain"],
|
|
297
|
+
limit=10000,
|
|
298
|
+
)
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
---
|
|
302
|
+
|
|
303
|
+
### `client.propagation_geometry`
|
|
304
|
+
|
|
305
|
+
Accepts a GeoJSON dict **or** a Shapely geometry:
|
|
306
|
+
|
|
307
|
+
```python
|
|
308
|
+
from shapely.geometry import Polygon
|
|
309
|
+
|
|
310
|
+
area = Polygon([
|
|
311
|
+
(-1.35, 50.65), (-1.1, 50.65),
|
|
312
|
+
(-1.1, 50.78), (-1.35, 50.78),
|
|
313
|
+
(-1.35, 50.65),
|
|
314
|
+
])
|
|
315
|
+
|
|
316
|
+
gdf = client.propagation_geometry(area)
|
|
317
|
+
|
|
318
|
+
# with options
|
|
319
|
+
gdf = client.propagation_geometry(
|
|
320
|
+
area,
|
|
321
|
+
layers=["buildings", "vegetation", "terrain"],
|
|
322
|
+
limit=50000,
|
|
323
|
+
)
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
---
|
|
327
|
+
|
|
328
|
+
### `client.features_bbox`
|
|
329
|
+
|
|
330
|
+
```python
|
|
331
|
+
gdf = client.features_bbox(
|
|
332
|
+
min_lon=-1.35,
|
|
333
|
+
min_lat=50.65,
|
|
334
|
+
max_lon=-1.1,
|
|
335
|
+
max_lat=50.78,
|
|
336
|
+
)
|
|
337
|
+
# gdf.columns → geometry, osm_id, name, source_layer, tags
|
|
338
|
+
```
|
|
339
|
+
|
|
340
|
+
---
|
|
341
|
+
|
|
342
|
+
## Data layers
|
|
343
|
+
|
|
344
|
+
| Layer | Source OSM tags | Key properties |
|
|
345
|
+
|---|---|---|
|
|
346
|
+
| `buildings` | `building=*` | `building_type`, `height_m`, `levels`, `material` |
|
|
347
|
+
| `roads` | `highway=*`, `railway=*` | `transport_class` (major / secondary / local / rail / …) |
|
|
348
|
+
| `water` | `natural=water`, `waterway=*`, `landuse=reservoir` | `water_class` |
|
|
349
|
+
| `vegetation` | `landuse=forest`, `natural=wood/scrub/heath/grassland`, `leisure=park` | `vegetation_class` |
|
|
350
|
+
| `landuse` | `landuse=*` | `landuse_class` (residential / industrial / agricultural / …) |
|
|
351
|
+
| `structures` | `man_made=mast/tower`, `power=tower/line`, bridges, embankments | `structure_class` |
|
|
352
|
+
| `terrain` | `natural=cliff/ridge/coastline/peak/saddle`, barriers, chimneys | `terrain_class`, `height_m` |
|
|
353
|
+
|
|
354
|
+
### Height estimation
|
|
355
|
+
|
|
356
|
+
`height_m` on buildings is derived from OSM tags in order of preference:
|
|
357
|
+
|
|
358
|
+
1. `height=*` (metres, if numeric)
|
|
359
|
+
2. `building:levels=*` × 3 m per floor
|
|
360
|
+
3. `NULL` if neither is tagged
|
|
361
|
+
|
|
362
|
+
### Terrain classes
|
|
363
|
+
|
|
364
|
+
| `terrain_class` | Geometry | Propagation relevance |
|
|
365
|
+
|---|---|---|
|
|
366
|
+
| `cliff` | line | Hard diffraction edge |
|
|
367
|
+
| `ridge` | line | Diffraction / shadowing |
|
|
368
|
+
| `coastline` | line | Land/sea boundary — sea has near-zero attenuation |
|
|
369
|
+
| `barrier_wall` | line | Urban canyon obstruction |
|
|
370
|
+
| `embankment` / `cutting` | line | Signal blocking |
|
|
371
|
+
| `peak` / `saddle` | point | Elevation proxy |
|
|
372
|
+
| `chimney` / `storage_tank` | point | Tall point obstacles |
|
|
373
|
+
| `communications_tower` | point | Co-channel interference sources |
|
|
374
|
+
|
osmforge-1.1.2/README.md
ADDED
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
# OSMForge
|
|
2
|
+
|
|
3
|
+
A self-hosted OpenStreetMap stack that downloads regional OSM extracts, ingests
|
|
4
|
+
them into a local PostGIS database, and serves a GeoJSON API optimised for
|
|
5
|
+
spatial analysis — particularly **radio-frequency propagation modelling**.
|
|
6
|
+
|
|
7
|
+
Public OSM APIs (Overpass, Nominatim) have strict rate limits and aren't suited
|
|
8
|
+
to bulk or repeated spatial queries. OSMForge gives you unlimited local access
|
|
9
|
+
to the same data.
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Contents
|
|
14
|
+
|
|
15
|
+
- [Architecture](#architecture)
|
|
16
|
+
- [Prerequisites](#prerequisites)
|
|
17
|
+
- [Quick start](#quick-start)
|
|
18
|
+
- [Adding regions](#adding-regions)
|
|
19
|
+
- [Makefile reference](#makefile-reference)
|
|
20
|
+
- [API reference](#api-reference)
|
|
21
|
+
- [Python client](#python-client)
|
|
22
|
+
- [Data layers](#data-layers)
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## Architecture
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
Geofabrik ──download──▶ *.osm.pbf
|
|
30
|
+
│
|
|
31
|
+
osm2pgsql (Lua flex)
|
|
32
|
+
│
|
|
33
|
+
osm schema (raw)
|
|
34
|
+
osm_points / osm_lines / osm_polygons
|
|
35
|
+
│
|
|
36
|
+
make layers (SQL)
|
|
37
|
+
│
|
|
38
|
+
app schema (classified)
|
|
39
|
+
osm_buildings / osm_roads / osm_water /
|
|
40
|
+
osm_vegetation / osm_landuse /
|
|
41
|
+
osm_structures / osm_terrain
|
|
42
|
+
│
|
|
43
|
+
FastAPI :8000
|
|
44
|
+
│
|
|
45
|
+
osmforge.client
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The `osm` schema holds raw data exactly as loaded by osm2pgsql.
|
|
49
|
+
The `app` schema holds pre-classified, indexed tables ready for querying.
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Prerequisites
|
|
54
|
+
|
|
55
|
+
| Tool | Version | Notes |
|
|
56
|
+
|---|---|---|
|
|
57
|
+
| Docker + Compose v2 | latest | `docker compose version` |
|
|
58
|
+
| osm2pgsql | ≥ 1.9 | `osm2pgsql --version` |
|
|
59
|
+
| Python | ≥ 3.12 | |
|
|
60
|
+
| Poetry | ≥ 1.8 | `pip install poetry` |
|
|
61
|
+
|
|
62
|
+
Install Python dependencies:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
poetry install
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## Quick start
|
|
71
|
+
|
|
72
|
+
### 1 — Start the database
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
make up
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
This starts a PostGIS 17 container on `localhost:5432` and the FastAPI container
|
|
79
|
+
on `localhost:8000`. On first run Docker will pull the images.
|
|
80
|
+
|
|
81
|
+
### 2 — Download an OSM extract
|
|
82
|
+
|
|
83
|
+
Extracts come from [Geofabrik](https://download.geofabrik.de). Pass the full
|
|
84
|
+
path from the Geofabrik URL:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
make download REGION=europe/united-kingdom/england/isle-of-wight
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
The file lands in `osmforge/data/` and is skipped on subsequent runs unless you
|
|
91
|
+
pass `FORCE=--force`.
|
|
92
|
+
|
|
93
|
+
### 3 — Ingest + build layers
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
make ingest # load every *.osm.pbf in osmforge/data/ into osm.*
|
|
97
|
+
make layers # build classified app.* tables
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Or in one step:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
make rebuild
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### 4 — Query the API
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
curl "http://localhost:8000/propagation/bbox?min_lon=-1.35&min_lat=50.65&max_lon=-1.1&max_lat=50.78"
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## Adding regions
|
|
115
|
+
|
|
116
|
+
Download as many regions as you like before ingesting — `make ingest` loads
|
|
117
|
+
everything in `osmforge/data/` in a single pass.
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
# individual counties
|
|
121
|
+
make download REGION=europe/united-kingdom/england/west-midlands
|
|
122
|
+
make download REGION=europe/united-kingdom/england/staffordshire
|
|
123
|
+
make download REGION=europe/united-kingdom/england/warwickshire
|
|
124
|
+
|
|
125
|
+
# then ingest them all together
|
|
126
|
+
make rebuild
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
To see what Geofabrik has available, browse:
|
|
130
|
+
- England: https://download.geofabrik.de/europe/united-kingdom/england.html
|
|
131
|
+
- Full index: https://download.geofabrik.de
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## Makefile reference
|
|
136
|
+
|
|
137
|
+
| Command | Description |
|
|
138
|
+
|---|---|
|
|
139
|
+
| `make up` | Start PostGIS + API containers |
|
|
140
|
+
| `make down` | Stop containers (data volume preserved) |
|
|
141
|
+
| `make download REGION=<path>` | Download a PBF from Geofabrik |
|
|
142
|
+
| `make ingest` | Load all PBFs in `osmforge/data/` via osm2pgsql |
|
|
143
|
+
| `make layers` | Build `app.*` tables from raw `osm.*` data |
|
|
144
|
+
| `make rebuild` | `ingest` + `layers` in one step |
|
|
145
|
+
| `make rebuild-api` | Rebuild the API Docker image after code changes |
|
|
146
|
+
| `make clean` | Destroy containers **and** data volume (full reset) |
|
|
147
|
+
| `make interface` | Open a psql shell to the database |
|
|
148
|
+
| `make help` | List all targets |
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
|
|
152
|
+
## API reference
|
|
153
|
+
|
|
154
|
+
The API runs at `http://localhost:8000`. Interactive docs at
|
|
155
|
+
`http://localhost:8000/docs`.
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
### `GET /health`
|
|
160
|
+
|
|
161
|
+
```
|
|
162
|
+
200 {"status": "ok"}
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
---
|
|
166
|
+
|
|
167
|
+
### `GET /propagation/bbox`
|
|
168
|
+
|
|
169
|
+
Return classified OSM features within a bounding box.
|
|
170
|
+
|
|
171
|
+
**Query parameters**
|
|
172
|
+
|
|
173
|
+
| Parameter | Type | Required | Description |
|
|
174
|
+
|---|---|---|---|
|
|
175
|
+
| `min_lon` | float | ✓ | West edge (WGS-84) |
|
|
176
|
+
| `min_lat` | float | ✓ | South edge |
|
|
177
|
+
| `max_lon` | float | ✓ | East edge |
|
|
178
|
+
| `max_lat` | float | ✓ | North edge |
|
|
179
|
+
| `layers` | string (repeat) | | Subset of layers (default: all) |
|
|
180
|
+
| `limit` | int | | Max features returned |
|
|
181
|
+
|
|
182
|
+
**Example**
|
|
183
|
+
|
|
184
|
+
```
|
|
185
|
+
GET /propagation/bbox?min_lon=-1.35&min_lat=50.65&max_lon=-1.1&max_lat=50.78&layers=buildings&layers=terrain
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
**Response** — GeoJSON FeatureCollection. Each feature's `properties`:
|
|
189
|
+
|
|
190
|
+
```json
|
|
191
|
+
{
|
|
192
|
+
"osm_id": 123456,
|
|
193
|
+
"name": "St Mary's Church",
|
|
194
|
+
"layer": "buildings",
|
|
195
|
+
"feature_class": "church",
|
|
196
|
+
"height_m": 18.0,
|
|
197
|
+
"levels": "3",
|
|
198
|
+
"material": "stone"
|
|
199
|
+
}
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
---
|
|
203
|
+
|
|
204
|
+
### `POST /propagation/geometry`
|
|
205
|
+
|
|
206
|
+
Return classified OSM features intersecting an arbitrary polygon or
|
|
207
|
+
multipolygon. Useful when you already have a coverage or study-area polygon.
|
|
208
|
+
|
|
209
|
+
**Request body**
|
|
210
|
+
|
|
211
|
+
```json
|
|
212
|
+
{
|
|
213
|
+
"geometry": {
|
|
214
|
+
"type": "Polygon",
|
|
215
|
+
"coordinates": [[[<lon>, <lat>], ...]]
|
|
216
|
+
},
|
|
217
|
+
"layers": ["buildings", "vegetation", "terrain"],
|
|
218
|
+
"limit": 5000
|
|
219
|
+
}
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
| Field | Type | Required | Description |
|
|
223
|
+
|---|---|---|---|
|
|
224
|
+
| `geometry` | GeoJSON geometry | ✓ | Polygon or MultiPolygon, WGS-84 |
|
|
225
|
+
| `layers` | array of strings | | Subset of layers (default: all) |
|
|
226
|
+
| `limit` | int | | Max features returned |
|
|
227
|
+
|
|
228
|
+
---
|
|
229
|
+
|
|
230
|
+
### `GET /features/bbox`
|
|
231
|
+
|
|
232
|
+
Return **raw** OSM features (all tags) within a bounding box. Useful for
|
|
233
|
+
exploration; use `/propagation/bbox` for modelling workflows.
|
|
234
|
+
|
|
235
|
+
**Query parameters** — same bbox params as above, plus optional `limit`.
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
## Python client
|
|
240
|
+
|
|
241
|
+
Install the package then import the client:
|
|
242
|
+
|
|
243
|
+
```python
|
|
244
|
+
from osmforge.client import OSMClient
|
|
245
|
+
|
|
246
|
+
client = OSMClient() # default: http://localhost:8000
|
|
247
|
+
# or: OSMClient("http://my-server:8000")
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
All methods return a `geopandas.GeoDataFrame` (CRS EPSG:4326).
|
|
251
|
+
|
|
252
|
+
---
|
|
253
|
+
|
|
254
|
+
### `client.propagation_bbox`
|
|
255
|
+
|
|
256
|
+
```python
|
|
257
|
+
gdf = client.propagation_bbox(
|
|
258
|
+
min_lon=-1.35,
|
|
259
|
+
min_lat=50.65,
|
|
260
|
+
max_lon=-1.1,
|
|
261
|
+
max_lat=50.78,
|
|
262
|
+
)
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
Filter to specific layers:
|
|
266
|
+
|
|
267
|
+
```python
|
|
268
|
+
gdf = client.propagation_bbox(
|
|
269
|
+
min_lon=-1.35, min_lat=50.65,
|
|
270
|
+
max_lon=-1.1, max_lat=50.78,
|
|
271
|
+
layers=["buildings", "terrain"],
|
|
272
|
+
limit=10000,
|
|
273
|
+
)
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
---
|
|
277
|
+
|
|
278
|
+
### `client.propagation_geometry`
|
|
279
|
+
|
|
280
|
+
Accepts a GeoJSON dict **or** a Shapely geometry:
|
|
281
|
+
|
|
282
|
+
```python
|
|
283
|
+
from shapely.geometry import Polygon
|
|
284
|
+
|
|
285
|
+
area = Polygon([
|
|
286
|
+
(-1.35, 50.65), (-1.1, 50.65),
|
|
287
|
+
(-1.1, 50.78), (-1.35, 50.78),
|
|
288
|
+
(-1.35, 50.65),
|
|
289
|
+
])
|
|
290
|
+
|
|
291
|
+
gdf = client.propagation_geometry(area)
|
|
292
|
+
|
|
293
|
+
# with options
|
|
294
|
+
gdf = client.propagation_geometry(
|
|
295
|
+
area,
|
|
296
|
+
layers=["buildings", "vegetation", "terrain"],
|
|
297
|
+
limit=50000,
|
|
298
|
+
)
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
---
|
|
302
|
+
|
|
303
|
+
### `client.features_bbox`
|
|
304
|
+
|
|
305
|
+
```python
|
|
306
|
+
gdf = client.features_bbox(
|
|
307
|
+
min_lon=-1.35,
|
|
308
|
+
min_lat=50.65,
|
|
309
|
+
max_lon=-1.1,
|
|
310
|
+
max_lat=50.78,
|
|
311
|
+
)
|
|
312
|
+
# gdf.columns → geometry, osm_id, name, source_layer, tags
|
|
313
|
+
```
|
|
314
|
+
|
|
315
|
+
---
|
|
316
|
+
|
|
317
|
+
## Data layers
|
|
318
|
+
|
|
319
|
+
| Layer | Source OSM tags | Key properties |
|
|
320
|
+
|---|---|---|
|
|
321
|
+
| `buildings` | `building=*` | `building_type`, `height_m`, `levels`, `material` |
|
|
322
|
+
| `roads` | `highway=*`, `railway=*` | `transport_class` (major / secondary / local / rail / …) |
|
|
323
|
+
| `water` | `natural=water`, `waterway=*`, `landuse=reservoir` | `water_class` |
|
|
324
|
+
| `vegetation` | `landuse=forest`, `natural=wood/scrub/heath/grassland`, `leisure=park` | `vegetation_class` |
|
|
325
|
+
| `landuse` | `landuse=*` | `landuse_class` (residential / industrial / agricultural / …) |
|
|
326
|
+
| `structures` | `man_made=mast/tower`, `power=tower/line`, bridges, embankments | `structure_class` |
|
|
327
|
+
| `terrain` | `natural=cliff/ridge/coastline/peak/saddle`, barriers, chimneys | `terrain_class`, `height_m` |
|
|
328
|
+
|
|
329
|
+
### Height estimation
|
|
330
|
+
|
|
331
|
+
`height_m` on buildings is derived from OSM tags in order of preference:
|
|
332
|
+
|
|
333
|
+
1. `height=*` (metres, if numeric)
|
|
334
|
+
2. `building:levels=*` × 3 m per floor
|
|
335
|
+
3. `NULL` if neither is tagged
|
|
336
|
+
|
|
337
|
+
### Terrain classes
|
|
338
|
+
|
|
339
|
+
| `terrain_class` | Geometry | Propagation relevance |
|
|
340
|
+
|---|---|---|
|
|
341
|
+
| `cliff` | line | Hard diffraction edge |
|
|
342
|
+
| `ridge` | line | Diffraction / shadowing |
|
|
343
|
+
| `coastline` | line | Land/sea boundary — sea has near-zero attenuation |
|
|
344
|
+
| `barrier_wall` | line | Urban canyon obstruction |
|
|
345
|
+
| `embankment` / `cutting` | line | Signal blocking |
|
|
346
|
+
| `peak` / `saddle` | point | Elevation proxy |
|
|
347
|
+
| `chimney` / `storage_tank` | point | Tall point obstacles |
|
|
348
|
+
| `communications_tower` | point | Co-channel interference sources |
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Thin Python client for the local OSM API.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
from osmforge.client import OSMClient
|
|
6
|
+
|
|
7
|
+
client = OSMClient() # defaults to http://localhost:8000
|
|
8
|
+
|
|
9
|
+
# By bounding box
|
|
10
|
+
gdf = client.propagation_bbox(min_lon=-1.6, min_lat=50.55, max_lon=-1.0, max_lat=50.85)
|
|
11
|
+
|
|
12
|
+
# By arbitrary polygon / multipolygon (shapely or GeoJSON dict)
|
|
13
|
+
gdf = client.propagation_geometry(my_polygon)
|
|
14
|
+
|
|
15
|
+
# Filter to specific layers
|
|
16
|
+
gdf = client.propagation_geometry(my_polygon, layers=["buildings", "terrain"])
|
|
17
|
+
|
|
18
|
+
# Raw OSM tags (no classification)
|
|
19
|
+
gdf = client.features_bbox(min_lon=-1.6, min_lat=50.55, max_lon=-1.0, max_lat=50.85)
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from typing import Any, Dict, List, Optional, Union
|
|
25
|
+
|
|
26
|
+
import geopandas as gpd
|
|
27
|
+
import requests
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
from shapely.geometry import mapping
|
|
31
|
+
from shapely.geometry.base import BaseGeometry
|
|
32
|
+
_SHAPELY = True
|
|
33
|
+
except ImportError:
|
|
34
|
+
_SHAPELY = False
|
|
35
|
+
|
|
36
|
+
_GeoJSON = Dict[str, Any]
|
|
37
|
+
_Geometry = Union[_GeoJSON, "BaseGeometry"]
|
|
38
|
+
|
|
39
|
+
ALL_LAYERS = [
|
|
40
|
+
"buildings", "roads", "water",
|
|
41
|
+
"vegetation", "landuse", "structures", "terrain",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _to_geojson(geometry: _Geometry) -> _GeoJSON:
|
|
46
|
+
if isinstance(geometry, dict):
|
|
47
|
+
return geometry
|
|
48
|
+
if _SHAPELY and isinstance(geometry, BaseGeometry):
|
|
49
|
+
return mapping(geometry)
|
|
50
|
+
raise TypeError(
|
|
51
|
+
f"geometry must be a GeoJSON dict or a Shapely geometry, got {type(geometry)}"
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class OSMClient:
|
|
56
|
+
def __init__(self, base_url: str = "http://localhost:8000"):
|
|
57
|
+
self.base_url = base_url.rstrip("/")
|
|
58
|
+
|
|
59
|
+
def _get(self, path: str, params: dict) -> dict:
|
|
60
|
+
r = requests.get(f"{self.base_url}{path}", params=params, timeout=120)
|
|
61
|
+
r.raise_for_status()
|
|
62
|
+
return r.json()
|
|
63
|
+
|
|
64
|
+
def _post(self, path: str, body: dict) -> dict:
|
|
65
|
+
r = requests.post(f"{self.base_url}{path}", json=body, timeout=120)
|
|
66
|
+
r.raise_for_status()
|
|
67
|
+
return r.json()
|
|
68
|
+
|
|
69
|
+
@staticmethod
|
|
70
|
+
def _to_gdf(fc: dict) -> gpd.GeoDataFrame:
|
|
71
|
+
features = fc.get("features", [])
|
|
72
|
+
if not features:
|
|
73
|
+
return gpd.GeoDataFrame()
|
|
74
|
+
return gpd.GeoDataFrame.from_features(features, crs="EPSG:4326")
|
|
75
|
+
|
|
76
|
+
# ------------------------------------------------------------------
|
|
77
|
+
# Public methods
|
|
78
|
+
# ------------------------------------------------------------------
|
|
79
|
+
|
|
80
|
+
def propagation_bbox(
|
|
81
|
+
self,
|
|
82
|
+
min_lon: float,
|
|
83
|
+
min_lat: float,
|
|
84
|
+
max_lon: float,
|
|
85
|
+
max_lat: float,
|
|
86
|
+
layers: Optional[List[str]] = None,
|
|
87
|
+
limit: Optional[int] = None,
|
|
88
|
+
) -> gpd.GeoDataFrame:
|
|
89
|
+
"""
|
|
90
|
+
Return propagation-classified features within a bounding box.
|
|
91
|
+
|
|
92
|
+
Parameters
|
|
93
|
+
----------
|
|
94
|
+
min_lon, min_lat, max_lon, max_lat : float
|
|
95
|
+
Bounding box in WGS-84 degrees.
|
|
96
|
+
layers : list of str, optional
|
|
97
|
+
Subset of ALL_LAYERS to include. Defaults to all.
|
|
98
|
+
limit : int, optional
|
|
99
|
+
Maximum number of features to return.
|
|
100
|
+
"""
|
|
101
|
+
params: dict = {
|
|
102
|
+
"min_lon": min_lon, "min_lat": min_lat,
|
|
103
|
+
"max_lon": max_lon, "max_lat": max_lat,
|
|
104
|
+
}
|
|
105
|
+
for layer in (layers or ALL_LAYERS):
|
|
106
|
+
params.setdefault("layers", [])
|
|
107
|
+
params["layers"].append(layer)
|
|
108
|
+
if limit is not None:
|
|
109
|
+
params["limit"] = limit
|
|
110
|
+
return self._to_gdf(self._get("/propagation/bbox", params))
|
|
111
|
+
|
|
112
|
+
def propagation_geometry(
|
|
113
|
+
self,
|
|
114
|
+
geometry: _Geometry,
|
|
115
|
+
layers: Optional[List[str]] = None,
|
|
116
|
+
limit: Optional[int] = None,
|
|
117
|
+
) -> gpd.GeoDataFrame:
|
|
118
|
+
"""
|
|
119
|
+
Return propagation-classified features intersecting a polygon or multipolygon.
|
|
120
|
+
|
|
121
|
+
Parameters
|
|
122
|
+
----------
|
|
123
|
+
geometry : GeoJSON dict or Shapely geometry
|
|
124
|
+
The area of interest (Polygon or MultiPolygon).
|
|
125
|
+
layers : list of str, optional
|
|
126
|
+
Subset of ALL_LAYERS to include. Defaults to all.
|
|
127
|
+
limit : int, optional
|
|
128
|
+
Maximum number of features to return.
|
|
129
|
+
"""
|
|
130
|
+
body: dict = {"geometry": _to_geojson(geometry), "layers": layers or ALL_LAYERS}
|
|
131
|
+
if limit is not None:
|
|
132
|
+
body["limit"] = limit
|
|
133
|
+
return self._to_gdf(self._post("/propagation/geometry", body))
|
|
134
|
+
|
|
135
|
+
def features_bbox(
|
|
136
|
+
self,
|
|
137
|
+
min_lon: float,
|
|
138
|
+
min_lat: float,
|
|
139
|
+
max_lon: float,
|
|
140
|
+
max_lat: float,
|
|
141
|
+
limit: Optional[int] = None,
|
|
142
|
+
) -> gpd.GeoDataFrame:
|
|
143
|
+
"""
|
|
144
|
+
Return raw OSM features (all tags) within a bounding box.
|
|
145
|
+
|
|
146
|
+
Useful for exploration; use propagation_bbox for modelling.
|
|
147
|
+
"""
|
|
148
|
+
params: dict = {
|
|
149
|
+
"min_lon": min_lon, "min_lat": min_lat,
|
|
150
|
+
"max_lon": max_lon, "max_lat": max_lat,
|
|
151
|
+
}
|
|
152
|
+
if limit is not None:
|
|
153
|
+
params["limit"] = limit
|
|
154
|
+
return self._to_gdf(self._get("/features/bbox", params))
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
services:
|
|
2
|
+
db:
|
|
3
|
+
image: postgis/postgis:17-3.6-alpine
|
|
4
|
+
container_name: osm-postgis
|
|
5
|
+
environment:
|
|
6
|
+
POSTGRES_DB: osm
|
|
7
|
+
POSTGRES_USER: osm
|
|
8
|
+
POSTGRES_PASSWORD: osm
|
|
9
|
+
ports:
|
|
10
|
+
- "5432:5432"
|
|
11
|
+
volumes:
|
|
12
|
+
- osm_pgdata:/var/lib/postgresql/data
|
|
13
|
+
- ./db/init:/docker-entrypoint-initdb.d
|
|
14
|
+
healthcheck:
|
|
15
|
+
test: ["CMD-SHELL", "pg_isready -U osm -d osm"]
|
|
16
|
+
interval: 10s
|
|
17
|
+
timeout: 5s
|
|
18
|
+
retries: 10
|
|
19
|
+
|
|
20
|
+
api:
|
|
21
|
+
build: ./fastapi
|
|
22
|
+
container_name: osm-api
|
|
23
|
+
environment:
|
|
24
|
+
DATABASE_URL: postgresql+psycopg://osm:osm@db:5432/osm
|
|
25
|
+
ports:
|
|
26
|
+
- "8000:8000"
|
|
27
|
+
depends_on:
|
|
28
|
+
db:
|
|
29
|
+
condition: service_healthy
|
|
30
|
+
|
|
31
|
+
volumes:
|
|
32
|
+
osm_pgdata:
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Download OSM PBF extracts from Geofabrik.
|
|
4
|
+
|
|
5
|
+
Usage:
|
|
6
|
+
python3 osmforge/download.py <region-path> [region-path ...]
|
|
7
|
+
python3 osmforge/download.py --force <region-path>
|
|
8
|
+
|
|
9
|
+
Examples:
|
|
10
|
+
python3 osmforge/download.py europe/united-kingdom/england/isle-of-wight
|
|
11
|
+
python3 osmforge/download.py europe/united-kingdom/england/greater-london
|
|
12
|
+
python3 osmforge/download.py europe/united-kingdom/england/west-midlands \
|
|
13
|
+
europe/united-kingdom/england/staffordshire
|
|
14
|
+
|
|
15
|
+
Or via make:
|
|
16
|
+
make download REGION=europe/united-kingdom/england/isle-of-wight
|
|
17
|
+
make download REGION="europe/united-kingdom/england/west-midlands europe/united-kingdom/england/staffordshire"
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import os
|
|
21
|
+
import sys
|
|
22
|
+
import time
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
import requests
|
|
26
|
+
from platformdirs import user_data_dir
|
|
27
|
+
|
|
28
|
+
BASE_URL = "https://download.geofabrik.de"
|
|
29
|
+
INTER_FILE_DELAY = 2 # seconds between downloads
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def get_data_dir() -> Path:
|
|
33
|
+
"""
|
|
34
|
+
Resolve the directory where PBF files are stored.
|
|
35
|
+
|
|
36
|
+
Priority:
|
|
37
|
+
1. OSMFORGE_DATA_DIR environment variable
|
|
38
|
+
2. OS user-data directory (~/.local/share/osmforge on Linux,
|
|
39
|
+
~/Library/Application Support/osmforge on macOS, etc.)
|
|
40
|
+
"""
|
|
41
|
+
env = os.environ.get("OSMFORGE_DATA_DIR")
|
|
42
|
+
path = Path(env) if env else Path(user_data_dir("osmforge"))
|
|
43
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
44
|
+
return path
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def dest_path(region_path: str, data_dir: Path) -> Path:
|
|
48
|
+
leaf = region_path.rstrip("/").split("/")[-1]
|
|
49
|
+
return data_dir / f"{leaf}-latest.osm.pbf"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def download(region_path: str, force: bool = False) -> Path:
|
|
53
|
+
data_dir = get_data_dir()
|
|
54
|
+
url = f"{BASE_URL}/{region_path}-latest.osm.pbf"
|
|
55
|
+
dest = dest_path(region_path, data_dir)
|
|
56
|
+
|
|
57
|
+
if dest.exists() and not force:
|
|
58
|
+
size_mb = dest.stat().st_size / 1_048_576
|
|
59
|
+
print(f" [skip] {dest.name} already exists ({size_mb:.1f} MB)")
|
|
60
|
+
return dest
|
|
61
|
+
|
|
62
|
+
print(f" -> {url}")
|
|
63
|
+
print(f" saving to {dest}")
|
|
64
|
+
|
|
65
|
+
tmp = dest.with_suffix(".part")
|
|
66
|
+
try:
|
|
67
|
+
with requests.get(url, stream=True, timeout=60) as r:
|
|
68
|
+
r.raise_for_status()
|
|
69
|
+
total = int(r.headers.get("content-length", 0))
|
|
70
|
+
downloaded = 0
|
|
71
|
+
with tmp.open("wb") as f:
|
|
72
|
+
for chunk in r.iter_content(chunk_size=1 << 20):
|
|
73
|
+
f.write(chunk)
|
|
74
|
+
downloaded += len(chunk)
|
|
75
|
+
if total:
|
|
76
|
+
pct = downloaded / total * 100
|
|
77
|
+
mb = downloaded / 1_048_576
|
|
78
|
+
print(
|
|
79
|
+
f"\r {mb:6.1f} / {total/1_048_576:.1f} MB"
|
|
80
|
+
f" ({pct:.0f}%)",
|
|
81
|
+
end="",
|
|
82
|
+
flush=True,
|
|
83
|
+
)
|
|
84
|
+
print()
|
|
85
|
+
tmp.rename(dest)
|
|
86
|
+
print(f" saved -> {dest.name}")
|
|
87
|
+
except Exception:
|
|
88
|
+
tmp.unlink(missing_ok=True)
|
|
89
|
+
raise
|
|
90
|
+
|
|
91
|
+
return dest
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def main() -> None:
|
|
95
|
+
args = sys.argv[1:]
|
|
96
|
+
force = "--force" in args
|
|
97
|
+
regions = [a for a in args if not a.startswith("--")]
|
|
98
|
+
|
|
99
|
+
if not regions:
|
|
100
|
+
print(__doc__)
|
|
101
|
+
sys.exit(1)
|
|
102
|
+
|
|
103
|
+
print(f"Queued {len(regions)} region(s):\n")
|
|
104
|
+
for i, region in enumerate(regions):
|
|
105
|
+
print(f"[{i + 1}/{len(regions)}] {region}")
|
|
106
|
+
download(region, force=force)
|
|
107
|
+
if i < len(regions) - 1:
|
|
108
|
+
time.sleep(INTER_FILE_DELAY)
|
|
109
|
+
|
|
110
|
+
print("\nDone.")
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
if __name__ == "__main__":
|
|
114
|
+
main()
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
[tool.poetry]
|
|
2
|
+
name = "osmforge"
|
|
3
|
+
version = "1.1.2"
|
|
4
|
+
description = "Self-hosted OpenStreetMap stack and Python client for spatial analysis"
|
|
5
|
+
authors = ["Tom Freeman <tomrfreeman3@gmail.com>"]
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
homepage = "https://github.com/Tom3man/osm-forge"
|
|
9
|
+
repository = "https://github.com/Tom3man/osm-forge"
|
|
10
|
+
documentation = "https://tom3man.github.io/osm-forge/"
|
|
11
|
+
keywords = ["osm", "openstreetmap", "gis", "geospatial", "postgis", "propagation"]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 3 - Alpha",
|
|
14
|
+
"Intended Audience :: Developers",
|
|
15
|
+
"Intended Audience :: Science/Research",
|
|
16
|
+
"Topic :: Scientific/Engineering :: GIS",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
# Only ship the client-side modules; the server stack lives in the repo only.
|
|
23
|
+
packages = [{include = "osmforge"}]
|
|
24
|
+
exclude = [
|
|
25
|
+
"osmforge/db",
|
|
26
|
+
"osmforge/fastapi",
|
|
27
|
+
"osmforge/osm2pgsql",
|
|
28
|
+
"osmforge/data",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
[tool.poetry.dependencies]
|
|
33
|
+
# Runtime — installed when users `pip install osmforge`
|
|
34
|
+
python = "^3.12"
|
|
35
|
+
requests = "^2.32.5"
|
|
36
|
+
geopandas = "^1.1.1"
|
|
37
|
+
shapely = "^2.1.2"
|
|
38
|
+
platformdirs = "^4.0"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
[tool.poetry.group.dev.dependencies]
|
|
42
|
+
# Server stack + notebook tooling — not published to PyPI
|
|
43
|
+
fastapi = "^0.121.0"
|
|
44
|
+
uvicorn = "^0.38.0"
|
|
45
|
+
psycopg = {extras = ["binary"], version = "^3.0"}
|
|
46
|
+
click = "^8.3.0"
|
|
47
|
+
ipykernel = "^7.2.0"
|
|
48
|
+
matplotlib = "^3.10.8"
|
|
49
|
+
# Quality tooling
|
|
50
|
+
ruff = "^0.9"
|
|
51
|
+
mypy = "^1.11"
|
|
52
|
+
pytest = "^8.0"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
[tool.poetry.group.docs.dependencies]
|
|
56
|
+
mkdocs = "^1.6"
|
|
57
|
+
mkdocs-material = "^9.5"
|
|
58
|
+
mkdocstrings = {extras = ["python"], version = "^0.27"}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
[tool.poetry.scripts]
|
|
62
|
+
osmforge-download = "osmforge.download:main"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
[build-system]
|
|
66
|
+
requires = ["poetry-core"]
|
|
67
|
+
build-backend = "poetry.core.masonry.api"
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
[tool.ruff]
|
|
71
|
+
line-length = 100
|
|
72
|
+
target-version = "py312"
|
|
73
|
+
|
|
74
|
+
[tool.ruff.lint]
|
|
75
|
+
select = ["E", "F", "I"]
|
|
76
|
+
ignore = ["E501"]
|
|
77
|
+
|
|
78
|
+
[tool.mypy]
|
|
79
|
+
python_version = "3.12"
|
|
80
|
+
ignore_missing_imports = true
|
|
81
|
+
strict = false
|
|
82
|
+
|
|
83
|
+
[tool.pytest.ini_options]
|
|
84
|
+
testpaths = ["tests"]
|