tilegrab 1.1.0__py3-none-any.whl → 1.2.0b2__py3-none-any.whl

tilegrab/__init__.py

@@ -1,4 +1,4 @@
-version = "1.1.0"
+__version__ = "1.2.0b2"
 # from tilegrab import downloader
 # from tilegrab import sources
 # from tilegrab import mosaic

tilegrab/cli.py

@@ -1,95 +1,22 @@
 #!/usr/bin/env python3
 import logging
 import argparse
-import random
-import sys
 from pathlib import Path
+from typing import List
 from tilegrab.downloader import Downloader
-from tilegrab.images import TileImageCollection
+from tilegrab.images import TileImageCollection, ExportType
+from tilegrab.logs import setup_logging
 from tilegrab.tiles import TilesByShape, TilesByBBox
 from tilegrab.dataset import GeoDataset
-from tilegrab import version
-
-# Configure root logger
-# logging.basicConfig(
-#     level=logging.INFO,
-#     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-#     handlers=[
-#         logging.StreamHandler(sys.stdout),
-#         logging.FileHandler('tilegrab.log')
-#     ]
-# )
-
-# Normal colors
-BLACK   = "\033[30m"
-RED     = "\033[31m"
-GREEN   = "\033[32m"
-YELLOW  = "\033[33m"
-BLUE    = "\033[34m"
-MAGENTA = "\033[35m"
-CYAN    = "\033[36m"
-GRAY    = "\033[90m"
-WHITE  = "\033[37m"
-
-# Bright colors
-BBLACK   = "\033[90m"
-BRED     = "\033[91m"
-BGREEN   = "\033[92m"
-BYELLOW  = "\033[93m"
-BBLUE    = "\033[94m"
-BMAGENTA = "\033[95m"
-BCYAN    = "\033[96m"
-BGRAY    = "\033[97m"
-BWHITE = "\033[97m"
-
-RESET = "\033[0m"
-
-
-class LogFormatter(logging.Formatter):
-    NAME_WIDTH = 14
-
-    LEVEL_MAP = {
-        logging.CRITICAL: f'{RED}‼ {RESET}',
-        logging.ERROR:    f'{RED}✖ {RESET}',
-        logging.WARNING:  f'{YELLOW}⚠ {RESET}',
-        logging.INFO:     f'{BLUE}• {RESET}',
-        logging.DEBUG:    f'{GRAY}· {RESET}',
-        logging.NOTSET:   f'{CYAN}- {RESET}',
-    }
-
-    def format(self, record):
-        record.level_letter = self.LEVEL_MAP.get(record.levelno, '?')
-
-        short = record.name.rsplit('.', 1)[-1]
-        record.short_name = f"{short:<{self.NAME_WIDTH}}"
-
-        return super().format(record)
-
-
-console_formatter = LogFormatter(
-    f'   %(level_letter)s %(message)s'
-)
-file_formatter = logging.Formatter(
-    '%(asctime)s %(levelname)s %(name)s - %(message)s'
-)
-
-console = logging.StreamHandler(sys.stdout)
-console.setFormatter(console_formatter )
-
-file = logging.FileHandler('tilegrab.log')
-file.setFormatter(file_formatter)
-
-logging.basicConfig(
-    level=logging.INFO,
-    handlers=[console, file],
-)
+from tilegrab import __version__
 
-logger = logging.getLogger(__name__)
 
+logger = logging.getLogger(__name__)
 
 def parse_args() -> argparse.Namespace:
     p = argparse.ArgumentParser(
-        prog="tilegrab", description="Download and mosaic map tiles"
+        prog="tilegrab", 
+        description="Download and mosaic map tiles"
     )
 
     # Create a named group for the vector polygon source
@@ -127,13 +54,23 @@ def parse_args() -> argparse.Namespace:
         "--key", type=str, default=None, help="API key where required by source"
     )
 
+    # Create a named group for merged output format
+    mosaic_out_group = p.add_argument_group(
+        title="Mosaic export formats", description="Formats for the output mosaic image"
+    )
+    mosaic_group = mosaic_out_group.add_mutually_exclusive_group(required=False)
+    mosaic_group.add_argument("--jpg", action="store_true", help="JPG image; no geo-reference")
+    mosaic_group.add_argument("--png", action="store_true", help="PNG image; no geo-reference")
+    mosaic_group.add_argument("--tiff", action="store_true", help="GeoTiff image; with geo-reference")
+    mosaic_group.set_defaults(tiff=True)
+
     # other options
-    p.add_argument("--zoom", type=int, required=True, help="Zoom level (integer)")
+    p.add_argument("--zoom", type=int, required=True, help="Zoom level (integer between 1 and 22)")
     p.add_argument(
-        "--out",
+        "--tiles-out",
         type=Path,
         default=Path.cwd() / "saved_tiles",
-        help="Output directory (default: ./saved_tiles)",
+        help="Output directory for downloaded tiles (default: ./saved_tiles)",
     )
     p.add_argument(
         "--download-only",
@@ -146,63 +83,69 @@ def parse_args() -> argparse.Namespace:
         help="Only mosaic tiles; do not download",
     )
     p.add_argument(
-        "--no-progress", action="store_false", help="Hide download progress bar"
+        "--group-tiles", type=str, default=None, help="Mosaic tiles but according to given WxH into ./grouped_tiles"
     )
     p.add_argument(
-        "--quiet", action="store_true", help="Hide all prints"
+        "--group-overlap", action="store_true", help="Overlap with the next consecutive tile when grouping"
     )
     p.add_argument(
-        "--debug", action="store_true", help="Enable debug logging"
+        "--tile-limit", type=int, default=250, help="Override maximum tile limit that can download (use with caution)"
     )
     p.add_argument(
-        "--test", action="store_true", help="Only for testing purposes, not for normal use"
+        "--workers", type=int, default=None, help="Max number of threads to use when parallel downloading"
     )
-
+    p.add_argument(
+        "--parallel",
+        action=argparse.BooleanOptionalAction,
+        default=False, 
+        help="Download tiles sequentially, no parallel downloading"
+    )
+    p.add_argument(
+        "--progress", 
+        action=argparse.BooleanOptionalAction,
+        default=False, 
+        help="Hide tile download progress bar"
+    )
+    p.add_argument("--quiet", action="store_true", help="Hide all prints")
+    p.add_argument("--debug", action="store_true", help="Enable debug logging")
     return p.parse_args()
 
 
 def main():
+    LOG_LEVEL = logging.INFO
+    ENABLE_CLI_LOG = True
+    ENABLE_FILE_LOG = True
+
     args = parse_args()
-    
-    # Adjust logging level
     if args.debug:
-        logging.getLogger().setLevel(logging.DEBUG)
-        # logger.debug("Debug logging enabled")
-    elif args.quiet:
-        console.close()
-        
+        LOG_LEVEL = logging.DEBUG
+    if args.quiet:
+        ENABLE_CLI_LOG = False
     
-    
-    BANNER_NORMAL, BANNER_BRIGHT = random.choice([
-        (RED, BRED),
-        (GREEN, BGREEN),
-        (YELLOW, BYELLOW),
-        (BLUE, BBLUE),
-        (MAGENTA, BMAGENTA),
-        (CYAN, BCYAN),
-        (GRAY, BGRAY),
-    ])
+    setup_logging(ENABLE_CLI_LOG, ENABLE_FILE_LOG, LOG_LEVEL)
 
     if not args.quiet:
         print()
-        print(f"{WHITE}   " + ("-" * 60) + f"{RESET}")
-        print(f"{BWHITE}  TileGrab v{version}{RESET}".rjust(50))
-        print(f"{WHITE}   " + ("-" * 60) + f"{RESET}")
-    
+        print(f"\033[37m   " + ("-" * 60) + "\033[0m")
+        print(f"\033[97m  TileGrab v{__version__}\033[0m".rjust(50))
+        print(f"\033[37m   " + ("-" * 60) + "\033[0m")
+
     try:
         dataset = GeoDataset(args.source)
         logger.info(f"Dataset loaded successfully from {args.source}")
 
         _tmp = "bbox" if args.bbox else "shape" if args.shape else "DnE"
-        logger.info(f"""Downloading tiles using {_tmp}
+        logger.info(
+            f"""Downloading tiles using {_tmp}
         - minX: {dataset.bbox.minx:.4f}     - minY: {dataset.bbox.miny:.4f}
         - maxX: {dataset.bbox.maxx:.4f}     - maxY: {dataset.bbox.maxy:.4f}
-        - zoom: {args.zoom}""")
+        - zoom: {args.zoom}"""
+        )
 
         if args.shape:
-            tiles = TilesByShape(dataset, zoom=args.zoom)
+            tiles = TilesByShape(geo_dataset=dataset, zoom=args.zoom, SAFE_LIMIT=args.tile_limit)
         elif args.bbox:
-            tiles = TilesByBBox(dataset, zoom=args.zoom)
+            tiles = TilesByBBox(geo_dataset=dataset, zoom=args.zoom, SAFE_LIMIT=args.tile_limit)
         else:
             logger.error("No extent selector selected")
             raise SystemExit("No extent selector selected")
@@ -210,34 +153,65 @@ def main():
         # Choose source provider
         if args.osm:
             from tilegrab.sources import OSM
+
             logger.info("Using OpenStreetMap (OSM) as tile source")
             source = OSM(api_key=args.key) if args.key else OSM()
         elif args.google_sat:
             from tilegrab.sources import GoogleSat
+
             logger.info("Using Google Satellite as tile source")
             source = GoogleSat(api_key=args.key) if args.key else GoogleSat()
         elif args.esri_sat:
             from tilegrab.sources import ESRIWorldImagery
+
             logger.info("Using ESRI World Imagery as tile source")
-            source = ESRIWorldImagery(api_key=args.key) if args.key else ESRIWorldImagery()
+            source = (
+                ESRIWorldImagery(api_key=args.key) if args.key else ESRIWorldImagery()
+            )
         else:
             logger.error("No tile source selected")
             raise SystemExit("No tile source selected")
+
+        downloader = Downloader(
+            tile_collection=tiles, 
+            tile_source=source, 
+            temp_tile_dir=args.tiles_out)
         
-        downloader = Downloader(tiles, source, args.out)
         result: TileImageCollection
-
         if args.mosaic_only:
-            result = TileImageCollection(args.out)
-            result.load(tiles)  
+            result = TileImageCollection(path=args.tiles_out)
+            result.load(tile_collection=tiles)
         else:
-            result = downloader.run(show_progress=args.no_progress)
+            result = downloader.run(
+                workers=args.workers, 
+                show_progress=args.no_progress, 
+                parallel_download=args.no_parallel)
             logger.info(f"Download result: {result}")
+
+        ex_types: List[int] = []
         
-        if not args.download_only: result.mosaic()
+        if not args.download_only:
+            if args.tiff: 
+                ex_types.append(ExportType.TIFF)
+            if args.png: 
+                ex_types.append(ExportType.PNG)
+            if args.jpg: 
+                ex_types.append(ExportType.JPG)
+
+            if args.group_tiles:
+                w,h = args.group_tiles.lower().split("x")
+                result.group(
+                    width=int(w), 
+                    height=int(h),
+                    export_types=ex_types,
+                    overlap=args.group_overlap)
+            else:
+                result.mosaic(export_types=ex_types)
+            
+            
+
         logger.info("Done")
 
-        
     except Exception as e:
         logger.exception("Fatal error during execution")
         raise SystemExit(1)

tilegrab/dataset.py

@@ -1,14 +1,17 @@
+from dataclasses import dataclass
 import logging
 from pathlib import Path
 from box import Box
 from typing import Union
-
+from functools import cache
 
 logger = logging.getLogger(__name__)
 
-class GeoDataset:
+TILE_EPSG = 4326 #Web Mercator - 3857 | 4326 - WGS84
 
+class GeoDataset:
     @property
+    @cache
     def bbox(self):
         minx, miny, maxx, maxy = self.source.total_bounds
         bbox_dict = Box({"minx": minx, "miny": miny, "maxx": maxx, "maxy": maxy})
@@ -21,14 +24,14 @@ class GeoDataset:
     
     @property
     def x_extent(self):
-        minx, maxx = self.source.total_bounds
+        minx, _, maxx, _ = self.source.total_bounds
         extent = (maxx - minx) + 1
         logger.debug(f"X extent calculated: {extent}")
         return extent
 
     @property
     def y_extent(self):
-        miny, maxy = self.source.total_bounds
+        _, miny, _, maxy = self.source.total_bounds
         extent = (maxy - miny) + 1
         logger.debug(f"Y extent calculated: {extent}")
         return extent
@@ -63,14 +66,14 @@ class GeoDataset:
             logger.critical("Dataset has no CRS defined")
             raise RuntimeError("Missing CRS")
 
-        if epsg != 4326:
-            logger.info(f"Reprojecting dataset from EPSG:{epsg} to EPSG:4326")
-            gdf = gdf.to_crs(epsg=4326)
+        if epsg != TILE_EPSG: #Web Mercator
+            logger.info(f"Reprojecting dataset from EPSG:{epsg} to EPSG:{TILE_EPSG}")
+            gdf = gdf.to_crs(epsg=TILE_EPSG)
         else:
-            logger.debug("Dataset already in EPSG:4326")
+            logger.debug(f"Dataset already in EPSG:{TILE_EPSG}")
 
         self.original_epsg = epsg
-        self.current_epsg = 4326
+        self.current_epsg = TILE_EPSG
         self.source = gdf
         self.source_path = source_path
         logger.info(f"GeoDataset initialized successfully: {len(gdf)} features")

tilegrab/downloader.py

@@ -1,24 +1,20 @@
 import logging
 import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass
 from typing import List, Optional, Union
 import requests
-from requests.adapters import HTTPAdapter, Retry
-from tqdm import tqdm
-import tempfile
 from pathlib import Path
 
 from tilegrab.sources import TileSource
 from tilegrab.tiles import TileCollection, Tile
 from tilegrab.images import TileImageCollection, TileImage
-from PIL import Image
+
 
 logger = logging.getLogger(__name__)
 
 @dataclass
 class Downloader:
-    tiles: TileCollection
+    tile_collection: TileCollection
     tile_source: TileSource
     temp_tile_dir: Optional[Union[str, Path]] = None
     session: Optional[requests.Session] = None
@@ -29,6 +25,7 @@ class Downloader:
 
     def __post_init__(self):
         if not self.temp_tile_dir:
+            import tempfile
             tmpdir = tempfile.mkdtemp()
             self.temp_tile_dir = Path(tmpdir)
             logger.debug(f"Created temporary directory: {tmpdir}")
@@ -41,6 +38,8 @@ class Downloader:
         logger.info(f"Downloader initialized: source={self.tile_source.name}, timeout={self.REQUEST_TIMEOUT}s, max_retries={self.MAX_RETRIES}")
 
     def _init_session(self) -> requests.Session:
+        from requests.adapters import HTTPAdapter, Retry
+
         logger.debug("Initializing HTTP session with retry strategy")
         session = requests.Session()
         retries = Retry(
@@ -90,36 +89,41 @@ class Downloader:
 
     def run(
         self,
-        workers: int = 8,   
+        workers: Union[int, None] = None,   
         show_progress: bool = True,
+        parallel_download: bool = True
     ) -> TileImageCollection:
-        logger.info(f"Starting download run: {len(self.tiles)} tiles, workers={workers}, show_progress={show_progress}")
+        logger.info(f"Starting download run: {len(self.tile_collection)} tiles, workers={workers}, show_progress={show_progress}")
         
         results = []
 
         if show_progress:
-            pbar = tqdm(total=len(self.tiles), desc=f"Downloading", unit="tile")
+            from tqdm import tqdm
+            pbar = tqdm(total=len(self.tile_collection), desc=f"      Downloading", unit="tile")
         else:
             pbar = None
         
-        for tile in self.tiles.to_list:
-            res = self.download_tile(tile)
-            results.append(res)
-            if pbar:
-                pbar.update(1)
-        
-        # with ThreadPoolExecutor(max_workers=workers) as exe:
-        #     future_to_tile = {
-        #         exe.submit(self.download_tile, tile): tile for tile in self.tiles.to_list
-        #     }
-        #     for fut in as_completed(future_to_tile):
-        #         try:
-        #             results.append(fut.result())
-        #         except Exception:
-        #             results.append(False)
-
-        #         if pbar:
-        #             pbar.update(1)
+
+        if parallel_download:
+            from concurrent.futures import ThreadPoolExecutor, as_completed
+            with ThreadPoolExecutor(max_workers=workers) as exe:
+                future_to_tile = {
+                    exe.submit(self.download_tile, tile): tile for tile in self.tile_collection.to_list
+                }
+                for fut in as_completed(future_to_tile):
+                    try:
+                        results.append(fut.result())
+                    except Exception:
+                        results.append(False)
+                        
+                    if pbar:
+                        pbar.update(1)
+        else:
+            for tile in self.tile_collection.to_list:
+                res = self.download_tile(tile)
+                results.append(res)
+                if pbar:
+                    pbar.update(1)
 
         if pbar:
             pbar.close()
@@ -129,7 +133,7 @@ class Downloader:
 
     def _evaluate_result(self, result: List):
         success = sum(1 for v in result if v)
-        total = len(self.tiles)
+        total = len(self.tile_collection)
         logger.info(f"Download completed: {success}/{total} successful ({100*success/total:.1f}%)")
         if success < total:
             logger.warning(f"Failed to download {total - success} tiles")