tilefetch 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tilefetch-0.1.0/CHANGELOG.md +32 -0
- tilefetch-0.1.0/CITATION.cff +21 -0
- tilefetch-0.1.0/LICENSE +21 -0
- tilefetch-0.1.0/MANIFEST.in +8 -0
- tilefetch-0.1.0/PKG-INFO +264 -0
- tilefetch-0.1.0/README.md +224 -0
- tilefetch-0.1.0/pyproject.toml +131 -0
- tilefetch-0.1.0/setup.cfg +4 -0
- tilefetch-0.1.0/src/tilefetch/__init__.py +110 -0
- tilefetch-0.1.0/src/tilefetch/__main__.py +5 -0
- tilefetch-0.1.0/src/tilefetch/_version.py +1 -0
- tilefetch-0.1.0/src/tilefetch/api.py +592 -0
- tilefetch-0.1.0/src/tilefetch/cli.py +419 -0
- tilefetch-0.1.0/src/tilefetch/download.py +334 -0
- tilefetch-0.1.0/src/tilefetch/georef.py +291 -0
- tilefetch-0.1.0/src/tilefetch/mosaic.py +191 -0
- tilefetch-0.1.0/src/tilefetch/pmtiles.py +657 -0
- tilefetch-0.1.0/src/tilefetch/providers.py +137 -0
- tilefetch-0.1.0/src/tilefetch/py.typed +0 -0
- tilefetch-0.1.0/src/tilefetch/sources.py +546 -0
- tilefetch-0.1.0/src/tilefetch/storage.py +338 -0
- tilefetch-0.1.0/src/tilefetch/tiles.py +469 -0
- tilefetch-0.1.0/src/tilefetch.egg-info/PKG-INFO +264 -0
- tilefetch-0.1.0/src/tilefetch.egg-info/SOURCES.txt +36 -0
- tilefetch-0.1.0/src/tilefetch.egg-info/dependency_links.txt +1 -0
- tilefetch-0.1.0/src/tilefetch.egg-info/entry_points.txt +2 -0
- tilefetch-0.1.0/src/tilefetch.egg-info/requires.txt +11 -0
- tilefetch-0.1.0/src/tilefetch.egg-info/top_level.txt +1 -0
- tilefetch-0.1.0/tests/__init__.py +0 -0
- tilefetch-0.1.0/tests/conftest.py +307 -0
- tilefetch-0.1.0/tests/test_api.py +496 -0
- tilefetch-0.1.0/tests/test_cli.py +409 -0
- tilefetch-0.1.0/tests/test_download.py +189 -0
- tilefetch-0.1.0/tests/test_georef.py +106 -0
- tilefetch-0.1.0/tests/test_pmtiles.py +259 -0
- tilefetch-0.1.0/tests/test_storage.py +111 -0
- tilefetch-0.1.0/tests/test_tiles.py +227 -0
- tilefetch-0.1.0/tests/test_wmts.py +102 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
## [Unreleased]
|
|
6
|
+
|
|
7
|
+
## [0.1.0] - 2026-11-08
|
|
8
|
+
### Added
|
|
9
|
+
- `tilefetch.fetch`: download XYZ / TMS / WMTS tiles for a bbox or GeoJSON (Multi)Polygon at
|
|
10
|
+
one or more zoom levels into MBTiles, PMTiles v3, a `z/x/y` folder or a georeferenced image.
|
|
11
|
+
- `tilefetch.mosaic`: stitch one zoom level into a single image, cropped exactly to the bbox
|
|
12
|
+
(resampled), snapped to source pixels, or as whole tiles; optional EPSG:4326 output.
|
|
13
|
+
Works from the network or offline from an existing MBTiles / PMTiles / folder.
|
|
14
|
+
- `tilefetch.estimate`: tile counts per zoom, size and time estimates, mosaic pixel size,
|
|
15
|
+
limit check; optional sampling of real tile sizes.
|
|
16
|
+
- Tile math: Web Mercator projection, tile bounds, TMS row flip, quadkeys, antimeridian
|
|
17
|
+
bboxes, exact tile/polygon intersection with holes.
|
|
18
|
+
- Sources: URL templates with `{z} {x} {y} {-y} {q} {s} {bbox}` and custom parameters; WMTS
|
|
19
|
+
REST and KVP GetTile from GetCapabilities, with TileMatrixSet parsing (scale-based zoom
|
|
20
|
+
mapping, offset matrix sets, dimensions such as `Time`).
|
|
21
|
+
- Provider registry with terms URLs, attributions and hard per-job limits: USGS The National
|
|
22
|
+
Map (topo, imagery, shaded relief) and NASA GIBS (Blue Marble, Blue Marble relief, MODIS
|
|
23
|
+
Terra). Hosts that forbid bulk downloading (OSMF tile servers, OpenTopoMap, Google, Bing,
|
|
24
|
+
Esri ArcGIS Online) are refused.
|
|
25
|
+
- Downloader: 2 threads by default, shared rate limit, retries with exponential back-off and
|
|
26
|
+
`Retry-After`, timeouts, on-disk cache with ETag / Last-Modified revalidation, resume,
|
|
27
|
+
honest `tilefetch/<version>` User-Agent (browser strings refused).
|
|
28
|
+
- Writers: MBTiles 1.3 (TMS rows, full metadata); PMTiles v3 (Hilbert IDs, varint
|
|
29
|
+
directories, gzip, leaf directories, de-duplication, run-lengths) with a reader; Pillow-only
|
|
30
|
+
GeoTIFF with GeoKeys for EPSG:3857 / EPSG:4326 or via rasterio; PNG/JPEG/WebP with world
|
|
31
|
+
file and `.prj`.
|
|
32
|
+
- CLI: `tilefetch providers | estimate | get | mosaic | info`.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this software, please cite it as below."
|
|
3
|
+
authors:
|
|
4
|
+
- family-names: "Eminoğlu"
|
|
5
|
+
given-names: "Yusuf"
|
|
6
|
+
email: "yusufeminoglu@gmail.com"
|
|
7
|
+
orcid: "https://orcid.org/0009-0005-6000-2934"
|
|
8
|
+
title: "tilefetch"
|
|
9
|
+
version: "0.1.0"
|
|
10
|
+
date-released: "2026-11-08"
|
|
11
|
+
license: "MIT"
|
|
12
|
+
repository-code: "https://github.com/YusufEminoglu/tilefetch_sdk"
|
|
13
|
+
url: "https://pypi.org/project/tilefetch/"
|
|
14
|
+
abstract: "Download XYZ/TMS/WMTS map tiles for an area into MBTiles, PMTiles v3, z/x/y folders or one georeferenced GeoTIFF/PNG, with provider-policy enforcement, polite concurrency, retries, caching and resume."
|
|
15
|
+
keywords:
|
|
16
|
+
- gis
|
|
17
|
+
- map tiles
|
|
18
|
+
- wmts
|
|
19
|
+
- mbtiles
|
|
20
|
+
- pmtiles
|
|
21
|
+
- geotiff
|
tilefetch-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Yusuf Eminoğlu
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
tilefetch-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tilefetch
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Download XYZ/TMS/WMTS map tiles for an area into MBTiles, PMTiles, z/x/y folders or one georeferenced GeoTIFF/PNG, politely and within provider terms.
|
|
5
|
+
Author-email: Yusuf Eminoğlu <yusufeminoglu@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/YusufEminoglu/tilefetch_sdk
|
|
8
|
+
Project-URL: Repository, https://github.com/YusufEminoglu/tilefetch_sdk
|
|
9
|
+
Project-URL: Issues, https://github.com/YusufEminoglu/tilefetch_sdk/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/YusufEminoglu/tilefetch_sdk/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: gis,tiles,xyz,tms,wmts,mbtiles,pmtiles,geotiff,web-mercator,basemap,imagery,offline-maps,remote-sensing,machine-learning,mosaic
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Operating System :: OS Independent
|
|
20
|
+
Classifier: Environment :: Console
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
23
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
24
|
+
Classifier: Intended Audience :: Science/Research
|
|
25
|
+
Classifier: Intended Audience :: Developers
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Python: >=3.9
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Requires-Dist: Pillow>=9.1.0
|
|
31
|
+
Provides-Extra: rasterio
|
|
32
|
+
Requires-Dist: rasterio>=1.3.0; extra == "rasterio"
|
|
33
|
+
Provides-Extra: dev
|
|
34
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
35
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
36
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
37
|
+
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
|
38
|
+
Requires-Dist: pmtiles>=3.4.0; extra == "dev"
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
<div align="center">
|
|
42
|
+
|
|
43
|
+
# tilefetch
|
|
44
|
+
|
|
45
|
+
[](https://pypi.org/project/tilefetch/)
|
|
46
|
+
[](https://pypi.org/project/tilefetch/)
|
|
47
|
+
[](https://github.com/YusufEminoglu/tilefetch_sdk/blob/main/LICENSE)
|
|
48
|
+
[](https://docs.astral.sh/ruff/)
|
|
49
|
+
|
|
50
|
+
**Download XYZ / TMS / WMTS map tiles for an area and get one georeferenced GeoTIFF/PNG,
|
|
51
|
+
an MBTiles file, a PMTiles v3 archive or a `z/x/y` folder - politely, resumably, and within
|
|
52
|
+
each provider's terms.**
|
|
53
|
+
|
|
54
|
+
[📦 PyPI](https://pypi.org/project/tilefetch/) • [💻 GitHub](https://github.com/YusufEminoglu/tilefetch_sdk) • [🐛 Issues](https://github.com/YusufEminoglu/tilefetch_sdk/issues)
|
|
55
|
+
|
|
56
|
+
</div>
|
|
57
|
+
|
|
58
|
+
`tilefetch` is for GIS and machine-learning work that needs a basemap or imagery for an
|
|
59
|
+
area: as **one image with correct georeferencing** (to drop into QGIS, rasterio or a
|
|
60
|
+
training pipeline), or as an **offline tile package**. It has a single required dependency
|
|
61
|
+
(Pillow) and writes GeoTIFF, MBTiles and PMTiles itself.
|
|
62
|
+
|
|
63
|
+
## ⚠️ Responsible use - read this first
|
|
64
|
+
|
|
65
|
+
Tile servers are run by people and organisations who pay for the bandwidth. Most free
|
|
66
|
+
community tile servers **forbid bulk or offline downloading**; for example the
|
|
67
|
+
[OSMF Tile Usage Policy](https://operations.osmfoundation.org/policies/tiles/) forbids it
|
|
68
|
+
for `tile.openstreetmap.org`. `tilefetch` is built to keep you inside the rules:
|
|
69
|
+
|
|
70
|
+
- **Known no-bulk hosts are refused.** URLs on `tile.openstreetmap.org`, `opentopomap.org`,
|
|
71
|
+
Google, Bing (`virtualearth.net`) and Esri's `arcgisonline.com` raise `PolicyError`
|
|
72
|
+
(see `tilefetch.RESTRICTED_HOSTS`). There is no override flag: run your own tile server, or
|
|
73
|
+
use a service whose terms allow offline use.
|
|
74
|
+
- **Built-in providers have hard limits.** Each entry carries its terms URL, its required
|
|
75
|
+
attribution, a maximum number of tiles per job, a request rate and a thread limit. You can
|
|
76
|
+
lower these, never raise them.
|
|
77
|
+
- **Custom URLs get conservative defaults**: 10,000 tiles per job and 10 requests/s. Raise the
|
|
78
|
+
cap (`max_tiles=` / `--max-tiles`) only for servers you operate or whose terms allow it.
|
|
79
|
+
- **Polite by default**: 2 threads, retries with exponential back-off that honour
|
|
80
|
+
`Retry-After`, an on-disk cache with `ETag` revalidation, and resume, so nothing is
|
|
81
|
+
downloaded twice.
|
|
82
|
+
- **Honest identification**: requests carry `User-Agent: tilefetch/<version> (+repo URL)`.
|
|
83
|
+
Browser User-Agent strings are refused; set your own application name and contact instead
|
|
84
|
+
(`user_agent="myproject/1.0 (me@example.org)"`).
|
|
85
|
+
- **Attribution travels with the data**: it is printed when a job starts and stored in
|
|
86
|
+
MBTiles/PMTiles metadata, GeoTIFF `Copyright`/`ImageDescription` tags and PNG text chunks.
|
|
87
|
+
Display it wherever you show the data.
|
|
88
|
+
|
|
89
|
+
You remain responsible for complying with the terms of every service you use.
|
|
90
|
+
|
|
91
|
+
## 📦 Installation
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
pip install tilefetch # Pillow only
|
|
95
|
+
pip install "tilefetch[rasterio]" # optional: write GeoTIFFs through GDAL/rasterio
|
|
96
|
+
pip install "tilefetch[dev]" # pytest, ruff, mypy, pmtiles (reference reader for tests)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Python 3.9+ on Windows, macOS and Linux.
|
|
100
|
+
|
|
101
|
+
## ⚡ Quickstart
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
import tilefetch
|
|
105
|
+
|
|
106
|
+
bbox = (-105.30, 39.95, -105.20, 40.05) # west, south, east, north (degrees)
|
|
107
|
+
|
|
108
|
+
# 1. How big is the job?
|
|
109
|
+
est = tilefetch.estimate("usgs-imagery", bbox, "12-15")
|
|
110
|
+
print(est.total_tiles, est.estimated_bytes, est.within_limit)
|
|
111
|
+
|
|
112
|
+
# 2. An offline package (MBTiles, PMTiles or a z/x/y folder, chosen by extension)
|
|
113
|
+
tilefetch.fetch("usgs-topo", bbox, "12-15", out="boulder.mbtiles")
|
|
114
|
+
tilefetch.fetch("usgs-topo", bbox, "12-15", out="boulder.pmtiles")
|
|
115
|
+
|
|
116
|
+
# 3. One georeferenced image cropped to exactly the bbox
|
|
117
|
+
res = tilefetch.mosaic("usgs-imagery", bbox, 15, out="boulder.tif")
|
|
118
|
+
print(res.image.size, res.bounds, res.crs) # EPSG:3857 bounds of the GeoTIFF
|
|
119
|
+
|
|
120
|
+
# ... or in EPSG:4326, as PNG + world file (.pgw) + .prj
|
|
121
|
+
tilefetch.mosaic("usgs-imagery", bbox, 15, out="boulder_4326.png", crs="EPSG:4326")
|
|
122
|
+
|
|
123
|
+
# 4. Your own tile server (any XYZ template)
|
|
124
|
+
tilefetch.fetch(
|
|
125
|
+
"https://tiles.example.org/{z}/{x}/{y}.png",
|
|
126
|
+
bbox, 14, out="area.pmtiles",
|
|
127
|
+
attribution="© My Org", user_agent="myproject/1.0 (me@example.org)",
|
|
128
|
+
)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
The same from the shell:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
tilefetch providers
|
|
135
|
+
tilefetch estimate -p usgs-imagery --bbox=-105.30,39.95,-105.20,40.05 -z 12-15
|
|
136
|
+
tilefetch get -p usgs-topo --bbox=-105.30,39.95,-105.20,40.05 -z 12-15 -o boulder.pmtiles
|
|
137
|
+
tilefetch get --url "https://tiles.example.org/{z}/{x}/{y}.png" \
|
|
138
|
+
--bbox=-105.3,39.95,-105.2,40.05 --zoom 12-15 -o out.pmtiles --attribution "© My Org"
|
|
139
|
+
tilefetch mosaic -p usgs-imagery --bbox=-105.30,39.95,-105.20,40.05 -z 15 -o out.tif
|
|
140
|
+
tilefetch mosaic --input boulder.mbtiles --bbox=-105.25,39.98,-105.22,40.0 -z 15 -o crop.png --crs EPSG:4326
|
|
141
|
+
tilefetch info boulder.pmtiles
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
Use `--bbox=...` (with `=`) when the first coordinate is negative, so it is not read as an option.
|
|
145
|
+
|
|
146
|
+
## 🗺️ Built-in providers
|
|
147
|
+
|
|
148
|
+
Only services whose published terms allow scripted/offline use are included.
|
|
149
|
+
|
|
150
|
+
| Name | Data | Zoom | Limits per job | Terms |
|
|
151
|
+
|---|---|---|---|---|
|
|
152
|
+
| `usgs-topo` | USGS Topo base map (US & territories) | 0-16 | 50,000 tiles, 8 req/s, 4 threads | [USGS: public domain, no restrictions](https://www.usgs.gov/faqs/what-are-terms-uselicensing-map-services-and-data-national-map) |
|
|
153
|
+
| `usgs-imagery` | USGS orthoimagery (US & territories) | 0-16 | same | same |
|
|
154
|
+
| `usgs-shaded-relief` | USGS shaded relief (US & territories) | 0-16 | same | same |
|
|
155
|
+
| `nasa-bluemarble` | NASA Blue Marble Next Generation (global, 500 m) | 0-8 | 100,000 tiles, 16 req/s, 8 threads | [NASA GIBS bulk-download guidance](https://nasa-gibs.github.io/gibs-api-docs/map-library-usage/#bulk-downloading) |
|
|
156
|
+
| `nasa-bluemarble-relief` | Blue Marble with shaded relief and bathymetry | 0-8 | same | same |
|
|
157
|
+
| `nasa-modis-terra` | MODIS Terra daily true colour (`params={"time": "YYYY-MM-DD"}`) | 0-9 | same | same |
|
|
158
|
+
|
|
159
|
+
- USGS asks for the credit *"Map services and data available from U.S. Geological Survey,
|
|
160
|
+
National Geospatial Program."*
|
|
161
|
+
- NASA GIBS defines a bulk download as more than 1,000,000 tiles in 24 hours and asks for
|
|
162
|
+
advance coordination above that; the per-job cap keeps one job far below it. Credit:
|
|
163
|
+
*"Imagery provided by services from NASA's Global Imagery Browse Services (GIBS), part of
|
|
164
|
+
NASA's Earth Science Data and Information System (ESDIS)."*
|
|
165
|
+
|
|
166
|
+
Register your own entries with `tilefetch.register_provider(TileSource(...))`.
|
|
167
|
+
|
|
168
|
+
## 🧩 Sources
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
from tilefetch import TileSource, wmts_source
|
|
172
|
+
|
|
173
|
+
# XYZ / TMS templates
|
|
174
|
+
TileSource.from_url("https://t.example.org/{z}/{x}/{y}.png") # XYZ
|
|
175
|
+
TileSource.from_url("https://t.example.org/{z}/{x}/{y}.png", scheme="tms") # TMS rows
|
|
176
|
+
TileSource.from_url("https://t.example.org/{z}/{x}/{-y}.png") # TMS row placeholder
|
|
177
|
+
TileSource.from_url("https://{s}.t.example.org/{z}/{x}/{y}.png", subdomains=("a", "b"))
|
|
178
|
+
TileSource.from_url("https://t.example.org/q/{q}.jpeg") # Bing-style quadkey
|
|
179
|
+
TileSource.from_url("https://wms.example.org/?SERVICE=WMS&REQUEST=GetMap&LAYERS=x"
|
|
180
|
+
"&SRS=EPSG:3857&BBOX={bbox}&WIDTH=256&HEIGHT=256&FORMAT=image/png")
|
|
181
|
+
TileSource.from_url("https://t.example.org/{z}/{x}/{y}.png?key={key}", params={"key": "..."})
|
|
182
|
+
|
|
183
|
+
# WMTS: REST (ResourceURL) or KVP GetTile, from a GetCapabilities URL, file or XML string
|
|
184
|
+
src = wmts_source(
|
|
185
|
+
"https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/1.0.0/WMTSCapabilities.xml",
|
|
186
|
+
"MODIS_Terra_CorrectedReflectance_TrueColor",
|
|
187
|
+
dimensions={"Time": "2024-06-01"},
|
|
188
|
+
attribution="NASA GIBS",
|
|
189
|
+
)
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
`wmts_source` parses the `TileMatrixSet`s, keeps only Web Mercator ones and maps each
|
|
193
|
+
`TileMatrix` to an XYZ zoom from its scale denominator (OGC 0.28 mm pixel), so matrix
|
|
194
|
+
identifiers such as `EPSG:3857:12`, 512-px tiles and regional matrix sets whose top-left
|
|
195
|
+
corner is offset from the world origin all work.
|
|
196
|
+
|
|
197
|
+
## 📤 Outputs
|
|
198
|
+
|
|
199
|
+
| Output | How | Notes |
|
|
200
|
+
|---|---|---|
|
|
201
|
+
| **MBTiles** | `out="x.mbtiles"` | MBTiles 1.3: `metadata` + `tiles` tables, rows in TMS order, `name`, `format`, `bounds`, `center`, `minzoom`, `maxzoom`, `attribution`, `description`, `type`, `version` |
|
|
202
|
+
| **PMTiles v3** | `out="x.pmtiles"` | 127-byte header, Hilbert tile IDs, varint directories, gzip internal compression, root directory in the first 16 KiB with leaf directories, de-duplicated tiles and run-lengths, clustered. Validated against the reference `pmtiles` reader |
|
|
203
|
+
| **Folder** | any other path | `z/x/y.png` (extension from the tile bytes) + `metadata.json` |
|
|
204
|
+
| **GeoTIFF** | `.tif` / `.tiff` | Pillow-only writer with ModelPixelScale, ModelTiepoint and GeoKeys for EPSG:3857 or EPSG:4326, deflate-compressed; or `engine="rasterio"` |
|
|
205
|
+
| **PNG / JPEG / WebP** | `.png` / `.jpg` / `.webp` | plus world file (`.pgw`/`.jgw`/`.wld`) and `.prj` (ESRI WKT) |
|
|
206
|
+
|
|
207
|
+
Mosaics (`tilefetch.mosaic` or an image `out=` in `fetch`) are built at one zoom level:
|
|
208
|
+
|
|
209
|
+
- `crop=True` (default) resamples to **exactly** the requested bbox (bilinear by default,
|
|
210
|
+
`resampling="nearest"`/`"bicubic"` available);
|
|
211
|
+
- `crop="snap"` crops to the enclosing source pixels with no resampling;
|
|
212
|
+
- `crop=False` keeps whole tiles;
|
|
213
|
+
- `crs="EPSG:4326"` resamples to a lon/lat grid with square-degree pixels; the inverse mapping
|
|
214
|
+
is exact per row (Mercator x is linear in longitude, each row's latitude is projected).
|
|
215
|
+
|
|
216
|
+
Missing tiles (HTTP 404/204) stay transparent. A mosaic can also be built offline from an
|
|
217
|
+
existing `.mbtiles`, `.pmtiles` or folder: `tilefetch.mosaic("area.mbtiles", bbox, 14, out=...)`.
|
|
218
|
+
|
|
219
|
+
## 🔁 Downloading behaviour
|
|
220
|
+
|
|
221
|
+
- `threads=2`, `rate=` requests/s (shared across threads), `retries=4`, `backoff=0.5` s
|
|
222
|
+
(exponential with jitter), `timeout=30` s.
|
|
223
|
+
- Retries on timeouts, connection errors and HTTP 408/425/429/500/502/503/504, honouring
|
|
224
|
+
`Retry-After`. 404/204/410 count as *missing*; 401/403 and other client errors fail at once.
|
|
225
|
+
An HTML/XML page served with status 200 is treated as a failure, not stored as a tile.
|
|
226
|
+
- `cache_dir=` keeps an HTTP cache (`ETag`, `Last-Modified`); `revalidate=True` sends
|
|
227
|
+
`If-None-Match` / `If-Modified-Since` and reuses the cached tile on `304 Not Modified`.
|
|
228
|
+
- **Resume**: re-running a job skips tiles already in the output. PMTiles jobs stage tiles in
|
|
229
|
+
`<out>.part` (SQLite) and write the archive at the end, so an interrupted job continues
|
|
230
|
+
where it stopped. `overwrite=True` starts again.
|
|
231
|
+
|
|
232
|
+
## 🧮 Tile math
|
|
233
|
+
|
|
234
|
+
```python
|
|
235
|
+
from tilefetch import (tiles_for_bbox, tiles_for_geometry, lonlat_to_tile, tile_bounds_mercator,
|
|
236
|
+
tile_to_quadkey, quadkey_to_tile, flip_y, zxy_to_tileid)
|
|
237
|
+
|
|
238
|
+
tiles_for_bbox((28.97, 41.00, 28.99, 41.02), "14-15") # antimeridian boxes (west > east) work
|
|
239
|
+
tiles_for_geometry("area.geojson", 14) # exact tile/polygon test, holes included
|
|
240
|
+
lonlat_to_tile(13.3777, 52.5163, 12) # Tile(z=12, x=2200, y=1343)
|
|
241
|
+
tile_to_quadkey((3, 3, 5)) # '213'
|
|
242
|
+
flip_y(3, 5) # 2 (XYZ <-> TMS)
|
|
243
|
+
zxy_to_tileid(1, 1, 0) # 4 (PMTiles Hilbert ID)
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
## 🧪 Development
|
|
247
|
+
|
|
248
|
+
```bash
|
|
249
|
+
git clone https://github.com/YusufEminoglu/tilefetch_sdk.git
|
|
250
|
+
cd tilefetch_sdk
|
|
251
|
+
pip install -e ".[dev,rasterio]"
|
|
252
|
+
python -m pytest --cov=tilefetch
|
|
253
|
+
python -m ruff check . && python -m ruff format --check src tests && python -m mypy src
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
The tests never touch the internet: they start an HTTP tile server on `127.0.0.1` that
|
|
257
|
+
serves generated tiles (and simulates 429/503/timeouts, ETags, WMTS REST/KVP and
|
|
258
|
+
capabilities). Mosaic tests check georeferencing pixel by pixel; GeoTIFFs are re-read with
|
|
259
|
+
rasterio when it is installed, and PMTiles archives with the reference `pmtiles` reader.
|
|
260
|
+
|
|
261
|
+
## 📄 License
|
|
262
|
+
|
|
263
|
+
MIT © Yusuf Eminoğlu. Map data and imagery downloaded with this tool remain under their
|
|
264
|
+
providers' licences and terms.
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
# tilefetch
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/tilefetch/)
|
|
6
|
+
[](https://pypi.org/project/tilefetch/)
|
|
7
|
+
[](https://github.com/YusufEminoglu/tilefetch_sdk/blob/main/LICENSE)
|
|
8
|
+
[](https://docs.astral.sh/ruff/)
|
|
9
|
+
|
|
10
|
+
**Download XYZ / TMS / WMTS map tiles for an area and get one georeferenced GeoTIFF/PNG,
|
|
11
|
+
an MBTiles file, a PMTiles v3 archive or a `z/x/y` folder - politely, resumably, and within
|
|
12
|
+
each provider's terms.**
|
|
13
|
+
|
|
14
|
+
[📦 PyPI](https://pypi.org/project/tilefetch/) • [💻 GitHub](https://github.com/YusufEminoglu/tilefetch_sdk) • [🐛 Issues](https://github.com/YusufEminoglu/tilefetch_sdk/issues)
|
|
15
|
+
|
|
16
|
+
</div>
|
|
17
|
+
|
|
18
|
+
`tilefetch` is for GIS and machine-learning work that needs a basemap or imagery for an
|
|
19
|
+
area: as **one image with correct georeferencing** (to drop into QGIS, rasterio or a
|
|
20
|
+
training pipeline), or as an **offline tile package**. It has a single required dependency
|
|
21
|
+
(Pillow) and writes GeoTIFF, MBTiles and PMTiles itself.
|
|
22
|
+
|
|
23
|
+
## ⚠️ Responsible use - read this first
|
|
24
|
+
|
|
25
|
+
Tile servers are run by people and organisations who pay for the bandwidth. Most free
|
|
26
|
+
community tile servers **forbid bulk or offline downloading**; for example the
|
|
27
|
+
[OSMF Tile Usage Policy](https://operations.osmfoundation.org/policies/tiles/) forbids it
|
|
28
|
+
for `tile.openstreetmap.org`. `tilefetch` is built to keep you inside the rules:
|
|
29
|
+
|
|
30
|
+
- **Known no-bulk hosts are refused.** URLs on `tile.openstreetmap.org`, `opentopomap.org`,
|
|
31
|
+
Google, Bing (`virtualearth.net`) and Esri's `arcgisonline.com` raise `PolicyError`
|
|
32
|
+
(see `tilefetch.RESTRICTED_HOSTS`). There is no override flag: run your own tile server, or
|
|
33
|
+
use a service whose terms allow offline use.
|
|
34
|
+
- **Built-in providers have hard limits.** Each entry carries its terms URL, its required
|
|
35
|
+
attribution, a maximum number of tiles per job, a request rate and a thread limit. You can
|
|
36
|
+
lower these, never raise them.
|
|
37
|
+
- **Custom URLs get conservative defaults**: 10,000 tiles per job and 10 requests/s. Raise the
|
|
38
|
+
cap (`max_tiles=` / `--max-tiles`) only for servers you operate or whose terms allow it.
|
|
39
|
+
- **Polite by default**: 2 threads, retries with exponential back-off that honour
|
|
40
|
+
`Retry-After`, an on-disk cache with `ETag` revalidation, and resume, so nothing is
|
|
41
|
+
downloaded twice.
|
|
42
|
+
- **Honest identification**: requests carry `User-Agent: tilefetch/<version> (+repo URL)`.
|
|
43
|
+
Browser User-Agent strings are refused; set your own application name and contact instead
|
|
44
|
+
(`user_agent="myproject/1.0 (me@example.org)"`).
|
|
45
|
+
- **Attribution travels with the data**: it is printed when a job starts and stored in
|
|
46
|
+
MBTiles/PMTiles metadata, GeoTIFF `Copyright`/`ImageDescription` tags and PNG text chunks.
|
|
47
|
+
Display it wherever you show the data.
|
|
48
|
+
|
|
49
|
+
You remain responsible for complying with the terms of every service you use.
|
|
50
|
+
|
|
51
|
+
## 📦 Installation
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install tilefetch # Pillow only
|
|
55
|
+
pip install "tilefetch[rasterio]" # optional: write GeoTIFFs through GDAL/rasterio
|
|
56
|
+
pip install "tilefetch[dev]" # pytest, ruff, mypy, pmtiles (reference reader for tests)
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Python 3.9+ on Windows, macOS and Linux.
|
|
60
|
+
|
|
61
|
+
## ⚡ Quickstart
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
import tilefetch
|
|
65
|
+
|
|
66
|
+
bbox = (-105.30, 39.95, -105.20, 40.05) # west, south, east, north (degrees)
|
|
67
|
+
|
|
68
|
+
# 1. How big is the job?
|
|
69
|
+
est = tilefetch.estimate("usgs-imagery", bbox, "12-15")
|
|
70
|
+
print(est.total_tiles, est.estimated_bytes, est.within_limit)
|
|
71
|
+
|
|
72
|
+
# 2. An offline package (MBTiles, PMTiles or a z/x/y folder, chosen by extension)
|
|
73
|
+
tilefetch.fetch("usgs-topo", bbox, "12-15", out="boulder.mbtiles")
|
|
74
|
+
tilefetch.fetch("usgs-topo", bbox, "12-15", out="boulder.pmtiles")
|
|
75
|
+
|
|
76
|
+
# 3. One georeferenced image cropped to exactly the bbox
|
|
77
|
+
res = tilefetch.mosaic("usgs-imagery", bbox, 15, out="boulder.tif")
|
|
78
|
+
print(res.image.size, res.bounds, res.crs) # EPSG:3857 bounds of the GeoTIFF
|
|
79
|
+
|
|
80
|
+
# ... or in EPSG:4326, as PNG + world file (.pgw) + .prj
|
|
81
|
+
tilefetch.mosaic("usgs-imagery", bbox, 15, out="boulder_4326.png", crs="EPSG:4326")
|
|
82
|
+
|
|
83
|
+
# 4. Your own tile server (any XYZ template)
|
|
84
|
+
tilefetch.fetch(
|
|
85
|
+
"https://tiles.example.org/{z}/{x}/{y}.png",
|
|
86
|
+
bbox, 14, out="area.pmtiles",
|
|
87
|
+
attribution="© My Org", user_agent="myproject/1.0 (me@example.org)",
|
|
88
|
+
)
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
The same from the shell:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
tilefetch providers
|
|
95
|
+
tilefetch estimate -p usgs-imagery --bbox=-105.30,39.95,-105.20,40.05 -z 12-15
|
|
96
|
+
tilefetch get -p usgs-topo --bbox=-105.30,39.95,-105.20,40.05 -z 12-15 -o boulder.pmtiles
|
|
97
|
+
tilefetch get --url "https://tiles.example.org/{z}/{x}/{y}.png" \
|
|
98
|
+
--bbox=-105.3,39.95,-105.2,40.05 --zoom 12-15 -o out.pmtiles --attribution "© My Org"
|
|
99
|
+
tilefetch mosaic -p usgs-imagery --bbox=-105.30,39.95,-105.20,40.05 -z 15 -o out.tif
|
|
100
|
+
tilefetch mosaic --input boulder.mbtiles --bbox=-105.25,39.98,-105.22,40.0 -z 15 -o crop.png --crs EPSG:4326
|
|
101
|
+
tilefetch info boulder.pmtiles
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Use `--bbox=...` (with `=`) when the first coordinate is negative, so it is not read as an option.
|
|
105
|
+
|
|
106
|
+
## 🗺️ Built-in providers
|
|
107
|
+
|
|
108
|
+
Only services whose published terms allow scripted/offline use are included.
|
|
109
|
+
|
|
110
|
+
| Name | Data | Zoom | Limits per job | Terms |
|
|
111
|
+
|---|---|---|---|---|
|
|
112
|
+
| `usgs-topo` | USGS Topo base map (US & territories) | 0-16 | 50,000 tiles, 8 req/s, 4 threads | [USGS: public domain, no restrictions](https://www.usgs.gov/faqs/what-are-terms-uselicensing-map-services-and-data-national-map) |
|
|
113
|
+
| `usgs-imagery` | USGS orthoimagery (US & territories) | 0-16 | same | same |
|
|
114
|
+
| `usgs-shaded-relief` | USGS shaded relief (US & territories) | 0-16 | same | same |
|
|
115
|
+
| `nasa-bluemarble` | NASA Blue Marble Next Generation (global, 500 m) | 0-8 | 100,000 tiles, 16 req/s, 8 threads | [NASA GIBS bulk-download guidance](https://nasa-gibs.github.io/gibs-api-docs/map-library-usage/#bulk-downloading) |
|
|
116
|
+
| `nasa-bluemarble-relief` | Blue Marble with shaded relief and bathymetry | 0-8 | same | same |
|
|
117
|
+
| `nasa-modis-terra` | MODIS Terra daily true colour (`params={"time": "YYYY-MM-DD"}`) | 0-9 | same | same |
|
|
118
|
+
|
|
119
|
+
- USGS asks for the credit *"Map services and data available from U.S. Geological Survey,
|
|
120
|
+
National Geospatial Program."*
|
|
121
|
+
- NASA GIBS defines a bulk download as more than 1,000,000 tiles in 24 hours and asks for
|
|
122
|
+
advance coordination above that; the per-job cap keeps one job far below it. Credit:
|
|
123
|
+
*"Imagery provided by services from NASA's Global Imagery Browse Services (GIBS), part of
|
|
124
|
+
NASA's Earth Science Data and Information System (ESDIS)."*
|
|
125
|
+
|
|
126
|
+
Register your own entries with `tilefetch.register_provider(TileSource(...))`.
|
|
127
|
+
|
|
128
|
+
## 🧩 Sources
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from tilefetch import TileSource, wmts_source
|
|
132
|
+
|
|
133
|
+
# XYZ / TMS templates
|
|
134
|
+
TileSource.from_url("https://t.example.org/{z}/{x}/{y}.png") # XYZ
|
|
135
|
+
TileSource.from_url("https://t.example.org/{z}/{x}/{y}.png", scheme="tms") # TMS rows
|
|
136
|
+
TileSource.from_url("https://t.example.org/{z}/{x}/{-y}.png") # TMS row placeholder
|
|
137
|
+
TileSource.from_url("https://{s}.t.example.org/{z}/{x}/{y}.png", subdomains=("a", "b"))
|
|
138
|
+
TileSource.from_url("https://t.example.org/q/{q}.jpeg") # Bing-style quadkey
|
|
139
|
+
TileSource.from_url("https://wms.example.org/?SERVICE=WMS&REQUEST=GetMap&LAYERS=x"
|
|
140
|
+
"&SRS=EPSG:3857&BBOX={bbox}&WIDTH=256&HEIGHT=256&FORMAT=image/png")
|
|
141
|
+
TileSource.from_url("https://t.example.org/{z}/{x}/{y}.png?key={key}", params={"key": "..."})
|
|
142
|
+
|
|
143
|
+
# WMTS: REST (ResourceURL) or KVP GetTile, from a GetCapabilities URL, file or XML string
|
|
144
|
+
src = wmts_source(
|
|
145
|
+
"https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/1.0.0/WMTSCapabilities.xml",
|
|
146
|
+
"MODIS_Terra_CorrectedReflectance_TrueColor",
|
|
147
|
+
dimensions={"Time": "2024-06-01"},
|
|
148
|
+
attribution="NASA GIBS",
|
|
149
|
+
)
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
`wmts_source` parses the `TileMatrixSet`s, keeps only Web Mercator ones and maps each
|
|
153
|
+
`TileMatrix` to an XYZ zoom from its scale denominator (OGC 0.28 mm pixel), so matrix
|
|
154
|
+
identifiers such as `EPSG:3857:12`, 512-px tiles and regional matrix sets whose top-left
|
|
155
|
+
corner is offset from the world origin all work.
|
|
156
|
+
|
|
157
|
+
## 📤 Outputs
|
|
158
|
+
|
|
159
|
+
| Output | How | Notes |
|
|
160
|
+
|---|---|---|
|
|
161
|
+
| **MBTiles** | `out="x.mbtiles"` | MBTiles 1.3: `metadata` + `tiles` tables, rows in TMS order, `name`, `format`, `bounds`, `center`, `minzoom`, `maxzoom`, `attribution`, `description`, `type`, `version` |
|
|
162
|
+
| **PMTiles v3** | `out="x.pmtiles"` | 127-byte header, Hilbert tile IDs, varint directories, gzip internal compression, root directory in the first 16 KiB with leaf directories, de-duplicated tiles and run-lengths, clustered. Validated against the reference `pmtiles` reader |
|
|
163
|
+
| **Folder** | any other path | `z/x/y.png` (extension from the tile bytes) + `metadata.json` |
|
|
164
|
+
| **GeoTIFF** | `.tif` / `.tiff` | Pillow-only writer with ModelPixelScale, ModelTiepoint and GeoKeys for EPSG:3857 or EPSG:4326, deflate-compressed; or `engine="rasterio"` |
|
|
165
|
+
| **PNG / JPEG / WebP** | `.png` / `.jpg` / `.webp` | plus world file (`.pgw`/`.jgw`/`.wld`) and `.prj` (ESRI WKT) |
|
|
166
|
+
|
|
167
|
+
Mosaics (`tilefetch.mosaic` or an image `out=` in `fetch`) are built at one zoom level:
|
|
168
|
+
|
|
169
|
+
- `crop=True` (default) resamples to **exactly** the requested bbox (bilinear by default,
|
|
170
|
+
`resampling="nearest"`/`"bicubic"` available);
|
|
171
|
+
- `crop="snap"` crops to the enclosing source pixels with no resampling;
|
|
172
|
+
- `crop=False` keeps whole tiles;
|
|
173
|
+
- `crs="EPSG:4326"` resamples to a lon/lat grid with square-degree pixels; the inverse mapping
|
|
174
|
+
is exact per row (Mercator x is linear in longitude, each row's latitude is projected).
|
|
175
|
+
|
|
176
|
+
Missing tiles (HTTP 404/204) stay transparent. A mosaic can also be built offline from an
|
|
177
|
+
existing `.mbtiles`, `.pmtiles` or folder: `tilefetch.mosaic("area.mbtiles", bbox, 14, out=...)`.
|
|
178
|
+
|
|
179
|
+
## 🔁 Downloading behaviour
|
|
180
|
+
|
|
181
|
+
- `threads=2`, `rate=` requests/s (shared across threads), `retries=4`, `backoff=0.5` s
|
|
182
|
+
(exponential with jitter), `timeout=30` s.
|
|
183
|
+
- Retries on timeouts, connection errors and HTTP 408/425/429/500/502/503/504, honouring
|
|
184
|
+
`Retry-After`. 404/204/410 count as *missing*; 401/403 and other client errors fail at once.
|
|
185
|
+
An HTML/XML page served with status 200 is treated as a failure, not stored as a tile.
|
|
186
|
+
- `cache_dir=` keeps an HTTP cache (`ETag`, `Last-Modified`); `revalidate=True` sends
|
|
187
|
+
`If-None-Match` / `If-Modified-Since` and reuses the cached tile on `304 Not Modified`.
|
|
188
|
+
- **Resume**: re-running a job skips tiles already in the output. PMTiles jobs stage tiles in
|
|
189
|
+
`<out>.part` (SQLite) and write the archive at the end, so an interrupted job continues
|
|
190
|
+
where it stopped. `overwrite=True` starts again.
|
|
191
|
+
|
|
192
|
+
## 🧮 Tile math
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
from tilefetch import (tiles_for_bbox, tiles_for_geometry, lonlat_to_tile, tile_bounds_mercator,
|
|
196
|
+
tile_to_quadkey, quadkey_to_tile, flip_y, zxy_to_tileid)
|
|
197
|
+
|
|
198
|
+
tiles_for_bbox((28.97, 41.00, 28.99, 41.02), "14-15") # antimeridian boxes (west > east) work
|
|
199
|
+
tiles_for_geometry("area.geojson", 14) # exact tile/polygon test, holes included
|
|
200
|
+
lonlat_to_tile(13.3777, 52.5163, 12) # Tile(z=12, x=2200, y=1343)
|
|
201
|
+
tile_to_quadkey((3, 3, 5)) # '213'
|
|
202
|
+
flip_y(3, 5) # 2 (XYZ <-> TMS)
|
|
203
|
+
zxy_to_tileid(1, 1, 0) # 4 (PMTiles Hilbert ID)
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
## 🧪 Development
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
git clone https://github.com/YusufEminoglu/tilefetch_sdk.git
|
|
210
|
+
cd tilefetch_sdk
|
|
211
|
+
pip install -e ".[dev,rasterio]"
|
|
212
|
+
python -m pytest --cov=tilefetch
|
|
213
|
+
python -m ruff check . && python -m ruff format --check src tests && python -m mypy src
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
The tests never touch the internet: they start an HTTP tile server on `127.0.0.1` that
|
|
217
|
+
serves generated tiles (and simulates 429/503/timeouts, ETags, WMTS REST/KVP and
|
|
218
|
+
capabilities). Mosaic tests check georeferencing pixel by pixel; GeoTIFFs are re-read with
|
|
219
|
+
rasterio when it is installed, and PMTiles archives with the reference `pmtiles` reader.
|
|
220
|
+
|
|
221
|
+
## 📄 License
|
|
222
|
+
|
|
223
|
+
MIT © Yusuf Eminoğlu. Map data and imagery downloaded with this tool remain under their
|
|
224
|
+
providers' licences and terms.
|