tilegrab 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tilegrab/__init__.py +14 -0
- tilegrab/__main__.py +3 -0
- tilegrab/cli.py +247 -0
- tilegrab/dataset.py +76 -0
- tilegrab/downloader.py +137 -0
- tilegrab/images.py +219 -0
- tilegrab/mosaic.py +75 -0
- tilegrab/sources.py +62 -0
- tilegrab/tiles.py +179 -0
- tilegrab-1.1.0.dist-info/METADATA +375 -0
- tilegrab-1.1.0.dist-info/RECORD +15 -0
- tilegrab-1.1.0.dist-info/WHEEL +5 -0
- tilegrab-1.1.0.dist-info/entry_points.txt +2 -0
- tilegrab-1.1.0.dist-info/licenses/LICENSE +21 -0
- tilegrab-1.1.0.dist-info/top_level.txt +1 -0
tilegrab/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
version = "1.1.0"
|
|
2
|
+
# from tilegrab import downloader
|
|
3
|
+
# from tilegrab import sources
|
|
4
|
+
# from tilegrab import mosaic
|
|
5
|
+
# from tilegrab import tiles
|
|
6
|
+
# from tilegrab import dataset
|
|
7
|
+
|
|
8
|
+
# __all__ = [
|
|
9
|
+
# "downloader",
|
|
10
|
+
# "dataset",
|
|
11
|
+
# "mosaic",
|
|
12
|
+
# "tiles",
|
|
13
|
+
# "sources",
|
|
14
|
+
# ]
|
tilegrab/__main__.py
ADDED
tilegrab/cli.py
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
import logging
|
|
3
|
+
import argparse
|
|
4
|
+
import random
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from tilegrab.downloader import Downloader
|
|
8
|
+
from tilegrab.images import TileImageCollection
|
|
9
|
+
from tilegrab.tiles import TilesByShape, TilesByBBox
|
|
10
|
+
from tilegrab.dataset import GeoDataset
|
|
11
|
+
from tilegrab import version
|
|
12
|
+
|
|
13
|
+
# Configure root logger
|
|
14
|
+
# logging.basicConfig(
|
|
15
|
+
# level=logging.INFO,
|
|
16
|
+
# format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
17
|
+
# handlers=[
|
|
18
|
+
# logging.StreamHandler(sys.stdout),
|
|
19
|
+
# logging.FileHandler('tilegrab.log')
|
|
20
|
+
# ]
|
|
21
|
+
# )
|
|
22
|
+
|
|
23
|
+
# Normal colors
|
|
24
|
+
BLACK = "\033[30m"
|
|
25
|
+
RED = "\033[31m"
|
|
26
|
+
GREEN = "\033[32m"
|
|
27
|
+
YELLOW = "\033[33m"
|
|
28
|
+
BLUE = "\033[34m"
|
|
29
|
+
MAGENTA = "\033[35m"
|
|
30
|
+
CYAN = "\033[36m"
|
|
31
|
+
GRAY = "\033[90m"
|
|
32
|
+
WHITE = "\033[37m"
|
|
33
|
+
|
|
34
|
+
# Bright colors
|
|
35
|
+
BBLACK = "\033[90m"
|
|
36
|
+
BRED = "\033[91m"
|
|
37
|
+
BGREEN = "\033[92m"
|
|
38
|
+
BYELLOW = "\033[93m"
|
|
39
|
+
BBLUE = "\033[94m"
|
|
40
|
+
BMAGENTA = "\033[95m"
|
|
41
|
+
BCYAN = "\033[96m"
|
|
42
|
+
BGRAY = "\033[97m"
|
|
43
|
+
BWHITE = "\033[97m"
|
|
44
|
+
|
|
45
|
+
RESET = "\033[0m"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class LogFormatter(logging.Formatter):
|
|
49
|
+
NAME_WIDTH = 14
|
|
50
|
+
|
|
51
|
+
LEVEL_MAP = {
|
|
52
|
+
logging.CRITICAL: f'{RED}‼ {RESET}',
|
|
53
|
+
logging.ERROR: f'{RED}✖ {RESET}',
|
|
54
|
+
logging.WARNING: f'{YELLOW}⚠ {RESET}',
|
|
55
|
+
logging.INFO: f'{BLUE}• {RESET}',
|
|
56
|
+
logging.DEBUG: f'{GRAY}· {RESET}',
|
|
57
|
+
logging.NOTSET: f'{CYAN}- {RESET}',
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
def format(self, record):
|
|
61
|
+
record.level_letter = self.LEVEL_MAP.get(record.levelno, '?')
|
|
62
|
+
|
|
63
|
+
short = record.name.rsplit('.', 1)[-1]
|
|
64
|
+
record.short_name = f"{short:<{self.NAME_WIDTH}}"
|
|
65
|
+
|
|
66
|
+
return super().format(record)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
console_formatter = LogFormatter(
|
|
70
|
+
f' %(level_letter)s %(message)s'
|
|
71
|
+
)
|
|
72
|
+
file_formatter = logging.Formatter(
|
|
73
|
+
'%(asctime)s %(levelname)s %(name)s - %(message)s'
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
console = logging.StreamHandler(sys.stdout)
|
|
77
|
+
console.setFormatter(console_formatter )
|
|
78
|
+
|
|
79
|
+
file = logging.FileHandler('tilegrab.log')
|
|
80
|
+
file.setFormatter(file_formatter)
|
|
81
|
+
|
|
82
|
+
logging.basicConfig(
|
|
83
|
+
level=logging.INFO,
|
|
84
|
+
handlers=[console, file],
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
logger = logging.getLogger(__name__)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def parse_args() -> argparse.Namespace:
|
|
91
|
+
p = argparse.ArgumentParser(
|
|
92
|
+
prog="tilegrab", description="Download and mosaic map tiles"
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
# Create a named group for the vector polygon source
|
|
96
|
+
extent_source_group = p.add_argument_group(
|
|
97
|
+
title="Source options(Extent)",
|
|
98
|
+
description="Options for the vector polygon source",
|
|
99
|
+
)
|
|
100
|
+
extent_source_group.add_argument(
|
|
101
|
+
"--source",
|
|
102
|
+
type=str,
|
|
103
|
+
required=True,
|
|
104
|
+
help="The vector polygon source for filter tiles",
|
|
105
|
+
)
|
|
106
|
+
extent_group = extent_source_group.add_mutually_exclusive_group(required=True)
|
|
107
|
+
extent_group.add_argument(
|
|
108
|
+
"--shape", action="store_true", help="Use actual shape to derive tiles"
|
|
109
|
+
)
|
|
110
|
+
extent_group.add_argument(
|
|
111
|
+
"--bbox", action="store_true", help="Use shape's bbox to derive tiles"
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
# Create a named group for the map tile source
|
|
115
|
+
tile_source_group = p.add_argument_group(
|
|
116
|
+
title="Source options(Map tiles)", description="Options for the map tile source"
|
|
117
|
+
)
|
|
118
|
+
tile_group = tile_source_group.add_mutually_exclusive_group(required=True)
|
|
119
|
+
tile_group.add_argument("--osm", action="store_true", help="OpenStreetMap")
|
|
120
|
+
tile_group.add_argument(
|
|
121
|
+
"--google_sat", action="store_true", help="Google Satellite"
|
|
122
|
+
)
|
|
123
|
+
tile_group.add_argument(
|
|
124
|
+
"--esri_sat", action="store_true", help="ESRI World Imagery"
|
|
125
|
+
)
|
|
126
|
+
tile_group.add_argument(
|
|
127
|
+
"--key", type=str, default=None, help="API key where required by source"
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
# other options
|
|
131
|
+
p.add_argument("--zoom", type=int, required=True, help="Zoom level (integer)")
|
|
132
|
+
p.add_argument(
|
|
133
|
+
"--out",
|
|
134
|
+
type=Path,
|
|
135
|
+
default=Path.cwd() / "saved_tiles",
|
|
136
|
+
help="Output directory (default: ./saved_tiles)",
|
|
137
|
+
)
|
|
138
|
+
p.add_argument(
|
|
139
|
+
"--download-only",
|
|
140
|
+
action="store_true",
|
|
141
|
+
help="Only download tiles; do not run mosaicking or postprocessing",
|
|
142
|
+
)
|
|
143
|
+
p.add_argument(
|
|
144
|
+
"--mosaic-only",
|
|
145
|
+
action="store_true",
|
|
146
|
+
help="Only mosaic tiles; do not download",
|
|
147
|
+
)
|
|
148
|
+
p.add_argument(
|
|
149
|
+
"--no-progress", action="store_false", help="Hide download progress bar"
|
|
150
|
+
)
|
|
151
|
+
p.add_argument(
|
|
152
|
+
"--quiet", action="store_true", help="Hide all prints"
|
|
153
|
+
)
|
|
154
|
+
p.add_argument(
|
|
155
|
+
"--debug", action="store_true", help="Enable debug logging"
|
|
156
|
+
)
|
|
157
|
+
p.add_argument(
|
|
158
|
+
"--test", action="store_true", help="Only for testing purposes, not for normal use"
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
return p.parse_args()
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def main():
|
|
165
|
+
args = parse_args()
|
|
166
|
+
|
|
167
|
+
# Adjust logging level
|
|
168
|
+
if args.debug:
|
|
169
|
+
logging.getLogger().setLevel(logging.DEBUG)
|
|
170
|
+
# logger.debug("Debug logging enabled")
|
|
171
|
+
elif args.quiet:
|
|
172
|
+
console.close()
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
BANNER_NORMAL, BANNER_BRIGHT = random.choice([
|
|
177
|
+
(RED, BRED),
|
|
178
|
+
(GREEN, BGREEN),
|
|
179
|
+
(YELLOW, BYELLOW),
|
|
180
|
+
(BLUE, BBLUE),
|
|
181
|
+
(MAGENTA, BMAGENTA),
|
|
182
|
+
(CYAN, BCYAN),
|
|
183
|
+
(GRAY, BGRAY),
|
|
184
|
+
])
|
|
185
|
+
|
|
186
|
+
if not args.quiet:
|
|
187
|
+
print()
|
|
188
|
+
print(f"{WHITE} " + ("-" * 60) + f"{RESET}")
|
|
189
|
+
print(f"{BWHITE} TileGrab v{version}{RESET}".rjust(50))
|
|
190
|
+
print(f"{WHITE} " + ("-" * 60) + f"{RESET}")
|
|
191
|
+
|
|
192
|
+
try:
|
|
193
|
+
dataset = GeoDataset(args.source)
|
|
194
|
+
logger.info(f"Dataset loaded successfully from {args.source}")
|
|
195
|
+
|
|
196
|
+
_tmp = "bbox" if args.bbox else "shape" if args.shape else "DnE"
|
|
197
|
+
logger.info(f"""Downloading tiles using {_tmp}
|
|
198
|
+
- minX: {dataset.bbox.minx:.4f} - minY: {dataset.bbox.miny:.4f}
|
|
199
|
+
- maxX: {dataset.bbox.maxx:.4f} - maxY: {dataset.bbox.maxy:.4f}
|
|
200
|
+
- zoom: {args.zoom}""")
|
|
201
|
+
|
|
202
|
+
if args.shape:
|
|
203
|
+
tiles = TilesByShape(dataset, zoom=args.zoom)
|
|
204
|
+
elif args.bbox:
|
|
205
|
+
tiles = TilesByBBox(dataset, zoom=args.zoom)
|
|
206
|
+
else:
|
|
207
|
+
logger.error("No extent selector selected")
|
|
208
|
+
raise SystemExit("No extent selector selected")
|
|
209
|
+
|
|
210
|
+
# Choose source provider
|
|
211
|
+
if args.osm:
|
|
212
|
+
from tilegrab.sources import OSM
|
|
213
|
+
logger.info("Using OpenStreetMap (OSM) as tile source")
|
|
214
|
+
source = OSM(api_key=args.key) if args.key else OSM()
|
|
215
|
+
elif args.google_sat:
|
|
216
|
+
from tilegrab.sources import GoogleSat
|
|
217
|
+
logger.info("Using Google Satellite as tile source")
|
|
218
|
+
source = GoogleSat(api_key=args.key) if args.key else GoogleSat()
|
|
219
|
+
elif args.esri_sat:
|
|
220
|
+
from tilegrab.sources import ESRIWorldImagery
|
|
221
|
+
logger.info("Using ESRI World Imagery as tile source")
|
|
222
|
+
source = ESRIWorldImagery(api_key=args.key) if args.key else ESRIWorldImagery()
|
|
223
|
+
else:
|
|
224
|
+
logger.error("No tile source selected")
|
|
225
|
+
raise SystemExit("No tile source selected")
|
|
226
|
+
|
|
227
|
+
downloader = Downloader(tiles, source, args.out)
|
|
228
|
+
result: TileImageCollection
|
|
229
|
+
|
|
230
|
+
if args.mosaic_only:
|
|
231
|
+
result = TileImageCollection(args.out)
|
|
232
|
+
result.load(tiles)
|
|
233
|
+
else:
|
|
234
|
+
result = downloader.run(show_progress=args.no_progress)
|
|
235
|
+
logger.info(f"Download result: {result}")
|
|
236
|
+
|
|
237
|
+
if not args.download_only: result.mosaic()
|
|
238
|
+
logger.info("Done")
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
except Exception as e:
|
|
242
|
+
logger.exception("Fatal error during execution")
|
|
243
|
+
raise SystemExit(1)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
if __name__ == "__main__":
|
|
247
|
+
main()
|
tilegrab/dataset.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from box import Box
|
|
4
|
+
from typing import Union
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
class GeoDataset:
|
|
10
|
+
|
|
11
|
+
@property
|
|
12
|
+
def bbox(self):
|
|
13
|
+
minx, miny, maxx, maxy = self.source.total_bounds
|
|
14
|
+
bbox_dict = Box({"minx": minx, "miny": miny, "maxx": maxx, "maxy": maxy})
|
|
15
|
+
logger.debug(f"Bbox calculated: minx={minx}, miny={miny}, maxx={maxx}, maxy={maxy}")
|
|
16
|
+
return bbox_dict
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
def shape(self):
|
|
20
|
+
return self.source.geometry
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def x_extent(self):
|
|
24
|
+
minx, maxx = self.source.total_bounds
|
|
25
|
+
extent = (maxx - minx) + 1
|
|
26
|
+
logger.debug(f"X extent calculated: {extent}")
|
|
27
|
+
return extent
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def y_extent(self):
|
|
31
|
+
miny, maxy = self.source.total_bounds
|
|
32
|
+
extent = (maxy - miny) + 1
|
|
33
|
+
logger.debug(f"Y extent calculated: {extent}")
|
|
34
|
+
return extent
|
|
35
|
+
|
|
36
|
+
def buffer(self, distance: int) -> None:
|
|
37
|
+
logger.debug(f"Buffering geometry by {distance} units")
|
|
38
|
+
self.source.geometry.buffer(distance)
|
|
39
|
+
|
|
40
|
+
def __init__(self, source_path: Union[Path, str]):
|
|
41
|
+
import geopandas as gpd
|
|
42
|
+
from pyproj import CRS
|
|
43
|
+
|
|
44
|
+
source_path = Path(source_path)
|
|
45
|
+
logger.info(f"Loading GeoDataset from: {source_path}")
|
|
46
|
+
|
|
47
|
+
try:
|
|
48
|
+
gdf = gpd.read_file(source_path)
|
|
49
|
+
except Exception as e:
|
|
50
|
+
logger.error(f"Failed to read geospatial file: {source_path}", exc_info=True)
|
|
51
|
+
raise
|
|
52
|
+
|
|
53
|
+
epsg = None
|
|
54
|
+
|
|
55
|
+
if gdf.crs is not None:
|
|
56
|
+
try:
|
|
57
|
+
epsg = CRS.from_user_input(gdf.crs).to_epsg()
|
|
58
|
+
logger.debug(f"Detected CRS EPSG code: {epsg}")
|
|
59
|
+
except Exception as e:
|
|
60
|
+
logger.critical(f"Unable to parse CRS from dataset: {gdf.crs}", exc_info=True)
|
|
61
|
+
raise RuntimeError("Unable to get CRS from the dataset")
|
|
62
|
+
else:
|
|
63
|
+
logger.critical("Dataset has no CRS defined")
|
|
64
|
+
raise RuntimeError("Missing CRS")
|
|
65
|
+
|
|
66
|
+
if epsg != 4326:
|
|
67
|
+
logger.info(f"Reprojecting dataset from EPSG:{epsg} to EPSG:4326")
|
|
68
|
+
gdf = gdf.to_crs(epsg=4326)
|
|
69
|
+
else:
|
|
70
|
+
logger.debug("Dataset already in EPSG:4326")
|
|
71
|
+
|
|
72
|
+
self.original_epsg = epsg
|
|
73
|
+
self.current_epsg = 4326
|
|
74
|
+
self.source = gdf
|
|
75
|
+
self.source_path = source_path
|
|
76
|
+
logger.info(f"GeoDataset initialized successfully: {len(gdf)} features")
|
tilegrab/downloader.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import List, Optional, Union
|
|
6
|
+
import requests
|
|
7
|
+
from requests.adapters import HTTPAdapter, Retry
|
|
8
|
+
from tqdm import tqdm
|
|
9
|
+
import tempfile
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from tilegrab.sources import TileSource
|
|
13
|
+
from tilegrab.tiles import TileCollection, Tile
|
|
14
|
+
from tilegrab.images import TileImageCollection, TileImage
|
|
15
|
+
from PIL import Image
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class Downloader:
|
|
21
|
+
tiles: TileCollection
|
|
22
|
+
tile_source: TileSource
|
|
23
|
+
temp_tile_dir: Optional[Union[str, Path]] = None
|
|
24
|
+
session: Optional[requests.Session] = None
|
|
25
|
+
REQUEST_TIMEOUT: int = 15
|
|
26
|
+
MAX_RETRIES: int = 5
|
|
27
|
+
BACKOFF_FACTOR: int = 0
|
|
28
|
+
OVERWRITE: bool = True
|
|
29
|
+
|
|
30
|
+
def __post_init__(self):
|
|
31
|
+
if not self.temp_tile_dir:
|
|
32
|
+
tmpdir = tempfile.mkdtemp()
|
|
33
|
+
self.temp_tile_dir = Path(tmpdir)
|
|
34
|
+
logger.debug(f"Created temporary directory: {tmpdir}")
|
|
35
|
+
else:
|
|
36
|
+
logger.debug(f"Using specified tile directory: {self.temp_tile_dir}")
|
|
37
|
+
|
|
38
|
+
os.makedirs(self.temp_tile_dir, exist_ok=True)
|
|
39
|
+
self.session = self.session or self._init_session()
|
|
40
|
+
self.image_col = TileImageCollection(self.temp_tile_dir)
|
|
41
|
+
logger.info(f"Downloader initialized: source={self.tile_source.name}, timeout={self.REQUEST_TIMEOUT}s, max_retries={self.MAX_RETRIES}")
|
|
42
|
+
|
|
43
|
+
def _init_session(self) -> requests.Session:
|
|
44
|
+
logger.debug("Initializing HTTP session with retry strategy")
|
|
45
|
+
session = requests.Session()
|
|
46
|
+
retries = Retry(
|
|
47
|
+
total=self.MAX_RETRIES,
|
|
48
|
+
backoff_factor=self.BACKOFF_FACTOR,
|
|
49
|
+
status_forcelist=[429, 500, 502, 503, 504],
|
|
50
|
+
allowed_methods=frozenset(["GET", "HEAD"]),
|
|
51
|
+
)
|
|
52
|
+
session.mount("https://", HTTPAdapter(max_retries=retries))
|
|
53
|
+
session.mount("http://", HTTPAdapter(max_retries=retries))
|
|
54
|
+
return session
|
|
55
|
+
|
|
56
|
+
def download_tile(self, tile: Tile) -> bool:
|
|
57
|
+
x, y, z = tile.x, tile.y, tile.z
|
|
58
|
+
url = self.tile_source.get_url(z, x, y)
|
|
59
|
+
headers = self.tile_source.headers() or {}
|
|
60
|
+
tile.url = url
|
|
61
|
+
|
|
62
|
+
logger.debug(f"Downloading tile: z={z}, x={x}, y={y}")
|
|
63
|
+
try:
|
|
64
|
+
resp = self.session.get(url, headers=headers, timeout=self.REQUEST_TIMEOUT) # type: ignore
|
|
65
|
+
resp.raise_for_status()
|
|
66
|
+
|
|
67
|
+
content_type = resp.headers.get("content-type", "")
|
|
68
|
+
if not content_type.startswith("image"):
|
|
69
|
+
logger.warning(f"Unexpected content type for z={z},x={x},y={y}: {content_type}")
|
|
70
|
+
raise ValueError(
|
|
71
|
+
f"Unexpected content type {z}/{x}/{y}: {content_type}"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
content = resp.content
|
|
75
|
+
if not content:
|
|
76
|
+
logger.warning(f"Empty content received for tile z={z},x={x},y={y}")
|
|
77
|
+
return False
|
|
78
|
+
|
|
79
|
+
img = TileImage(tile, content)
|
|
80
|
+
self.image_col.append(img)
|
|
81
|
+
logger.debug(f"Tile downloaded successfully: z={z}, x={x}, y={y}")
|
|
82
|
+
return True
|
|
83
|
+
|
|
84
|
+
except requests.exceptions.RequestException as e:
|
|
85
|
+
logger.warning(f"Failed to fetch tile z={z},x={x},y={y}: {str(e)}")
|
|
86
|
+
return False
|
|
87
|
+
except Exception as e:
|
|
88
|
+
logger.error(f"Unexpected error downloading z={z},x={x},y={y}", exc_info=True)
|
|
89
|
+
return False
|
|
90
|
+
|
|
91
|
+
def run(
|
|
92
|
+
self,
|
|
93
|
+
workers: int = 8,
|
|
94
|
+
show_progress: bool = True,
|
|
95
|
+
) -> TileImageCollection:
|
|
96
|
+
logger.info(f"Starting download run: {len(self.tiles)} tiles, workers={workers}, show_progress={show_progress}")
|
|
97
|
+
|
|
98
|
+
results = []
|
|
99
|
+
|
|
100
|
+
if show_progress:
|
|
101
|
+
pbar = tqdm(total=len(self.tiles), desc=f"Downloading", unit="tile")
|
|
102
|
+
else:
|
|
103
|
+
pbar = None
|
|
104
|
+
|
|
105
|
+
for tile in self.tiles.to_list:
|
|
106
|
+
res = self.download_tile(tile)
|
|
107
|
+
results.append(res)
|
|
108
|
+
if pbar:
|
|
109
|
+
pbar.update(1)
|
|
110
|
+
|
|
111
|
+
# with ThreadPoolExecutor(max_workers=workers) as exe:
|
|
112
|
+
# future_to_tile = {
|
|
113
|
+
# exe.submit(self.download_tile, tile): tile for tile in self.tiles.to_list
|
|
114
|
+
# }
|
|
115
|
+
# for fut in as_completed(future_to_tile):
|
|
116
|
+
# try:
|
|
117
|
+
# results.append(fut.result())
|
|
118
|
+
# except Exception:
|
|
119
|
+
# results.append(False)
|
|
120
|
+
|
|
121
|
+
# if pbar:
|
|
122
|
+
# pbar.update(1)
|
|
123
|
+
|
|
124
|
+
if pbar:
|
|
125
|
+
pbar.close()
|
|
126
|
+
|
|
127
|
+
self._evaluate_result(results)
|
|
128
|
+
return self.image_col
|
|
129
|
+
|
|
130
|
+
def _evaluate_result(self, result: List):
|
|
131
|
+
success = sum(1 for v in result if v)
|
|
132
|
+
total = len(self.tiles)
|
|
133
|
+
logger.info(f"Download completed: {success}/{total} successful ({100*success/total:.1f}%)")
|
|
134
|
+
if success < total:
|
|
135
|
+
logger.warning(f"Failed to download {total - success} tiles")
|
|
136
|
+
|
|
137
|
+
|
tilegrab/images.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from io import BytesIO
|
|
4
|
+
from pathlib import Path, PosixPath, WindowsPath
|
|
5
|
+
import re
|
|
6
|
+
from typing import Any, List, Optional, Tuple, Union
|
|
7
|
+
from PIL import Image as PLIImage
|
|
8
|
+
from box import Box
|
|
9
|
+
from tilegrab.tiles import Tile, TileCollection
|
|
10
|
+
import os
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
class ExportType:
|
|
15
|
+
PNG: int = 1
|
|
16
|
+
JPG: int = 2
|
|
17
|
+
TIFF: int = 3
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class TileImage:
|
|
21
|
+
width: int = 256
|
|
22
|
+
height: int = 256
|
|
23
|
+
|
|
24
|
+
def __init__(self, tile: Tile, image: Union[bytes, bytearray]) -> None:
|
|
25
|
+
self._tile = tile
|
|
26
|
+
try:
|
|
27
|
+
self._img = PLIImage.open(BytesIO(image))
|
|
28
|
+
logger.debug(f"TileImage created for z={tile.z},x={tile.x},y={tile.y}")
|
|
29
|
+
except Exception as e:
|
|
30
|
+
logger.error(f"Failed to open image for tile z={tile.z},x={tile.x},y={tile.y}", exc_info=True)
|
|
31
|
+
raise
|
|
32
|
+
|
|
33
|
+
self._path: Union[Path, None] = None
|
|
34
|
+
self._ext: str = self._get_image_type(image)
|
|
35
|
+
|
|
36
|
+
def __repr__(self) -> str:
|
|
37
|
+
return f"TileImage; name={self.name}; path={self.path}; url={self.url}; position={self.position}"
|
|
38
|
+
|
|
39
|
+
def _get_image_type(self, data: Union[bytes, bytearray]) -> str:
|
|
40
|
+
b = bytes(data)
|
|
41
|
+
|
|
42
|
+
# PNG: 8 bytes
|
|
43
|
+
if b.startswith(b'\x89PNG\r\n\x1a\n'):
|
|
44
|
+
logger.debug(f"Image detected as PNG")
|
|
45
|
+
return 'png'
|
|
46
|
+
|
|
47
|
+
# JPEG / JPG: files start with FF D8 and end with FF D9
|
|
48
|
+
if len(b) >= 2 and b[0:2] == b'\xff\xd8':
|
|
49
|
+
logger.debug(f"Image detected as JPG")
|
|
50
|
+
return 'jpg'
|
|
51
|
+
|
|
52
|
+
# BMP: starts with 'BM' (0x42 0x4D)
|
|
53
|
+
if len(b) >= 2 and b[0:2] == b'BM':
|
|
54
|
+
logger.debug(f"Image detected as BMP")
|
|
55
|
+
return 'bmp'
|
|
56
|
+
|
|
57
|
+
logger.warning(f"Unknown image format, defaulting to PNG")
|
|
58
|
+
return "png"
|
|
59
|
+
|
|
60
|
+
def save(self):
|
|
61
|
+
try:
|
|
62
|
+
self._img.save(self.path)
|
|
63
|
+
logger.debug(f"Image saved to {self.path}")
|
|
64
|
+
except Exception as e:
|
|
65
|
+
logger.error(f"Failed to save image to {self.path}", exc_info=True)
|
|
66
|
+
raise
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def name(self) -> str:
|
|
70
|
+
return f"{self._tile.z}_{self._tile.x}_{self._tile.y}.{self._ext}"
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def tile(self) -> Tile:
|
|
74
|
+
return self._tile
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def image(self) -> PLIImage.Image:
|
|
78
|
+
self._img.load()
|
|
79
|
+
return self._img
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def path(self) -> Path:
|
|
83
|
+
if self._path is None:
|
|
84
|
+
logger.error(f"Attempting to access path for unattached image")
|
|
85
|
+
raise RuntimeError("Image is not attached to a collection")
|
|
86
|
+
return self._path
|
|
87
|
+
|
|
88
|
+
@path.setter
|
|
89
|
+
def path(self, value: Any):
|
|
90
|
+
if isinstance(value, Path):
|
|
91
|
+
self._path = value
|
|
92
|
+
logger.debug(f"Image path set to {value}")
|
|
93
|
+
return
|
|
94
|
+
|
|
95
|
+
elif isinstance(value, str):
|
|
96
|
+
self._path = Path(value)
|
|
97
|
+
logger.debug(f"Image path set to {value}")
|
|
98
|
+
return
|
|
99
|
+
|
|
100
|
+
elif isinstance(value, WindowsPath):
|
|
101
|
+
self._path = Path(value)
|
|
102
|
+
logger.debug(f"Image path set to {value}")
|
|
103
|
+
return
|
|
104
|
+
|
|
105
|
+
elif isinstance(value, PosixPath):
|
|
106
|
+
self._path = Path(value)
|
|
107
|
+
logger.debug(f"Image path set to {value}")
|
|
108
|
+
return
|
|
109
|
+
|
|
110
|
+
logger.error(f"Invalid path type: {type(value)}")
|
|
111
|
+
raise TypeError(
|
|
112
|
+
"value must be a Path, WindowsPath, PosixPath, or path-like str"
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
@property
|
|
116
|
+
def extension(self) -> str:
|
|
117
|
+
if self._ext is None:
|
|
118
|
+
logger.error("Accessing extension for image without extension")
|
|
119
|
+
raise RuntimeError("Image does not have an extension")
|
|
120
|
+
return self._ext
|
|
121
|
+
|
|
122
|
+
@extension.setter
|
|
123
|
+
def extension(self, val: str):
|
|
124
|
+
self._ext = val
|
|
125
|
+
logger.debug(f"Image extension set to {val}")
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def position(self) -> Box:
|
|
129
|
+
return self._tile.position
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def url(self) -> Union[str, None]:
|
|
133
|
+
return self._tile.url
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class TileImageCollection:
|
|
137
|
+
images: List[TileImage] = []
|
|
138
|
+
width: int = 0
|
|
139
|
+
height: int = 0
|
|
140
|
+
|
|
141
|
+
def load(self, tile_collection:TileCollection):
|
|
142
|
+
logger.info("Start loading saved ImageTiles")
|
|
143
|
+
pat = re.compile(r'^([0-9]+)_([0-9]+)_([0-9]+)\.[A-Za-z0-9]+$')
|
|
144
|
+
image_col = [p for p in self.path.glob(f"*.*") if p.is_file()]
|
|
145
|
+
|
|
146
|
+
for tile in tile_collection.to_list:
|
|
147
|
+
found_matching_image = False
|
|
148
|
+
for image_path in image_col:
|
|
149
|
+
m = pat.match(str(image_path.name))
|
|
150
|
+
if m:
|
|
151
|
+
z = int(m.group(1))
|
|
152
|
+
x = int(m.group(2))
|
|
153
|
+
y = int(m.group(3))
|
|
154
|
+
|
|
155
|
+
if tile.x == x and tile.y == y and tile.z == z:
|
|
156
|
+
logger.debug(f"Processing ImageTile x={x} y={y} z={z}")
|
|
157
|
+
with open(image_path, 'rb') as f:
|
|
158
|
+
tile_image = TileImage(tile, f.read())
|
|
159
|
+
tile_image.path = image_path
|
|
160
|
+
self.images.append(tile_image)
|
|
161
|
+
found_matching_image = True
|
|
162
|
+
continue
|
|
163
|
+
|
|
164
|
+
if not found_matching_image: logger.warning(f"Missing ImageTile x={tile.x} y={tile.y} z={tile.z}")
|
|
165
|
+
|
|
166
|
+
def append(self, img: TileImage):
|
|
167
|
+
img.path = os.path.join(self.path, img.name)
|
|
168
|
+
self.images.append(img)
|
|
169
|
+
logger.debug(f"Image appended to collection: {img.name}")
|
|
170
|
+
img.save()
|
|
171
|
+
|
|
172
|
+
def __init__(self, path: Union[Path, str]) -> None:
|
|
173
|
+
self.path = Path(path)
|
|
174
|
+
logger.info(f"TileImageCollection initialized at {self.path}")
|
|
175
|
+
|
|
176
|
+
def __len__(self):
|
|
177
|
+
return sum(1 for _ in self)
|
|
178
|
+
|
|
179
|
+
def __iter__(self):
|
|
180
|
+
for i in self.images:
|
|
181
|
+
yield i
|
|
182
|
+
|
|
183
|
+
def __repr__(self) -> str:
|
|
184
|
+
return f"ImageCollection; len={len(self)}"
|
|
185
|
+
|
|
186
|
+
def _update_collection_dim(self):
|
|
187
|
+
if not self.images:
|
|
188
|
+
logger.warning("Attempting to update collection dimensions with no images")
|
|
189
|
+
return
|
|
190
|
+
|
|
191
|
+
x = [img.tile.x for img in self.images]
|
|
192
|
+
y = [img.tile.y for img in self.images]
|
|
193
|
+
minx, maxx = min(x), max(x)
|
|
194
|
+
miny, maxy = min(y), max(y)
|
|
195
|
+
|
|
196
|
+
self.width = int((maxx - minx + 1) * self.images[0].width)
|
|
197
|
+
self.height = int((maxy - miny + 1) * self.images[0].height)
|
|
198
|
+
logger.info(f"Collection dimensions calculated: {self.width}x{self.height}")
|
|
199
|
+
|
|
200
|
+
def mosaic(self):
|
|
201
|
+
logger.info("Starting mosaic creation")
|
|
202
|
+
self._update_collection_dim()
|
|
203
|
+
|
|
204
|
+
logger.info(f"Mosaicking {len(self.images)} images into {self.width}x{self.height}")
|
|
205
|
+
merged_image = PLIImage.new("RGB", (self.width, self.height))
|
|
206
|
+
|
|
207
|
+
for image in self.images:
|
|
208
|
+
px = int((image.position.x) * image.width)
|
|
209
|
+
py = int((image.position.y) * image.height)
|
|
210
|
+
logger.debug(f"Pasting image at position ({px}, {py}): {image.name}")
|
|
211
|
+
merged_image.paste(image.image, (px, py))
|
|
212
|
+
|
|
213
|
+
output_path = "merged_output.png"
|
|
214
|
+
merged_image.save(output_path)
|
|
215
|
+
logger.info(f"Mosaic saved to {output_path}")
|
|
216
|
+
|
|
217
|
+
def export_collection(self, type: ExportType):
|
|
218
|
+
logger.info(f"Exporting collection as type {type}")
|
|
219
|
+
pass
|