giga-spatial 0.6.4__py3-none-any.whl → 0.6.5__py3-none-any.whl

gigaspatial/generators/zonal/mercator.py

@@ -15,25 +15,48 @@ from gigaspatial.generators.zonal.geometry import GeometryBasedZonalViewGenerato
 
 
 class MercatorViewGenerator(GeometryBasedZonalViewGenerator[T]):
-    """Mid-level class for zonal view generation based on geometries with identifiers.
+    """
+    Generates zonal views using Mercator tiles as the zones.
 
-    This class serves as an intermediate between the abstract ZonalViewGenerator and specific
-    implementations like MercatorViewGenerator or H3ViewGenerator. It handles the common case
-    where zones are defined by a mapping between zone identifiers and geometries, either
-    provided as a dictionary or as a GeoDataFrame.
+    This class specializes in creating zonal views where the zones are defined by
+    Mercator tiles. It extends the `GeometryBasedZonalViewGenerator` and leverages
+    the `MercatorTiles` and `CountryMercatorTiles` classes to generate tiles based on
+    various input sources.
 
-    The class extends the base functionality with methods for mapping common geospatial
-    datasets including GHSL (Global Human Settlement Layer), Google Open Buildings,
-    and Microsoft Global Buildings data.
+    The primary input source defines the geographical area of interest. This can be
+    a country, a specific geometry, a set of points, or even a list of predefined
+    quadkeys. The `zoom_level` determines the granularity of the Mercator tiles.
 
     Attributes:
-        zone_dict (Dict[T, Polygon]): Mapping of zone identifiers to geometries.
-        zone_id_column (str): Name of the column containing zone identifiers.
-        zone_data_crs (str): Coordinate reference system of the zone data.
-        _zone_gdf (gpd.GeoDataFrame): Cached GeoDataFrame representation of zones.
-        data_store (DataStore): For accessing input data.
-        generator_config (ZonalViewGeneratorConfig): Configuration for view generation.
-        logger: Logger instance for this class.
+        source (Union[str, BaseGeometry, gpd.GeoDataFrame, List[Union[Point, Tuple[float, float]]], List[str]]):
+            Specifies the geographic area or specific tiles to use. Can be:
+            - A country name (str): Uses `CountryMercatorTiles` to generate tiles covering the country.
+            - A Shapely geometry (BaseGeometry):  Uses `MercatorTiles.from_spatial` to create tiles intersecting the geometry.
+            - A GeoDataFrame (gpd.GeoDataFrame): Uses `MercatorTiles.from_spatial` to create tiles intersecting the geometries.
+            - A list of points (List[Union[Point, Tuple[float, float]]]):  Uses `MercatorTiles.from_spatial` to create tiles containing the points.
+            - A list of quadkeys (List[str]): Uses `MercatorTiles.from_quadkeys` to use the specified tiles directly.
+        zoom_level (int): The zoom level of the Mercator tiles. Higher zoom levels result in smaller, more detailed tiles.
+        predicate (str):  The spatial predicate used when filtering tiles based on a spatial source (e.g., "intersects", "contains"). Defaults to "intersects".
+        config (Optional[ZonalViewGeneratorConfig]): Configuration for the zonal view generation process.
+        data_store (Optional[DataStore]):  A DataStore instance for accessing data.
+        logger (Optional[logging.Logger]):  A logger instance for logging.
+
+    Methods:
+        _init_zone_data(source, zoom_level, predicate):  Initializes the Mercator tile GeoDataFrame based on the input source.
+        # Inherits other methods from GeometryBasedZonalViewGenerator, such as:
+        # map_ghsl(), map_google_buildings(), map_ms_buildings(), aggregate_data(), save_view()
+
+    Example:
+        # Create a MercatorViewGenerator for tiles covering Germany at zoom level 6
+        generator = MercatorViewGenerator(source="Germany", zoom_level=6)
+
+        # Create a MercatorViewGenerator for tiles intersecting a specific polygon
+        polygon = ... # Define a Shapely Polygon
+        generator = MercatorViewGenerator(source=polygon, zoom_level=8)
+
+        # Create a MercatorViewGenerator from a list of quadkeys
+        quadkeys = ["0020023131023032", "0020023131023033"]
+        generator = MercatorViewGenerator(source=quadkeys, zoom_level=12)
     """
 
     def __init__(
@@ -53,16 +76,19 @@ class MercatorViewGenerator(GeometryBasedZonalViewGenerator[T]):
     ):
 
         super().__init__(
-            zone_data=self._init_zone_data(source, zoom_level, predicate),
+            zone_data=self._init_zone_data(source, zoom_level, predicate, data_store),
             zone_id_column="quadkey",
             config=config,
             data_store=data_store,
             logger=logger,
         )
+        self.logger.info(f"Initialized MercatorViewGenerator")
 
-    def _init_zone_data(self, source, zoom_level, predicate):
+    def _init_zone_data(self, source, zoom_level, predicate, data_store=None):
         if isinstance(source, str):
-            tiles = CountryMercatorTiles.create(country=source, zoom_level=zoom_level)
+            tiles = CountryMercatorTiles.create(
+                country=source, zoom_level=zoom_level, data_store=data_store
+            )
         elif isinstance(source, (BaseGeometry, Iterable)):
             if isinstance(source, Iterable) and all(
                 isinstance(qk, str) for qk in source
@@ -73,6 +99,11 @@ class MercatorViewGenerator(GeometryBasedZonalViewGenerator[T]):
                     source=source, zoom_level=zoom_level, predicate=predicate
                 )
         else:
-            raise ValueError("sadadasfasfkasmf")
+            raise TypeError(
+                f"Unsupported source type for MercatorViewGenerator. 'source' must be "
+                f"a country name (str), a Shapely geometry, a GeoDataFrame, "
+                f"a list of quadkeys (str), or a list of (lon, lat) tuples/Shapely Point objects. "
+                f"Received type: {type(source)}."
+            )
 
         return tiles.to_geodataframe()

gigaspatial/grid/__init__.py

@@ -1 +1 @@
-from gigaspatial.grid.mercator_tiles import *
+from gigaspatial.grid.mercator_tiles import MercatorTiles, CountryMercatorTiles

gigaspatial/grid/mercator_tiles.py

@@ -4,10 +4,10 @@ import mercantile
 from shapely.geometry import box
 from shapely.geometry.base import BaseGeometry
 from shapely.strtree import STRtree
-from shapely import MultiPolygon, Polygon, Point
+from shapely import Point
 import json
 from pathlib import Path
-from pydantic import BaseModel, Field, PrivateAttr
+from pydantic import BaseModel, Field
 from typing import List, Union, Iterable, Optional, Tuple, ClassVar
 import pycountry
 
@@ -31,6 +31,9 @@ class MercatorTiles(BaseModel):
         if not quadkeys:
             cls.logger.warning("No quadkeys provided to from_quadkeys.")
             return cls(zoom_level=0, quadkeys=[])
+        cls.logger.info(
+            f"Initializing MercatorTiles from {len(quadkeys)} provided quadkeys."
+        )
         return cls(zoom_level=len(quadkeys[0]), quadkeys=set(quadkeys))
 
     @classmethod
@@ -120,14 +123,7 @@ class MercatorTiles(BaseModel):
         cls.logger.info(
             f"Creating MercatorTiles from {len(points)} points at zoom level: {zoom_level}"
         )
-        quadkeys = {
-            (
-                mercantile.quadkey(mercantile.tile(p.x, p.y, zoom_level))
-                if isinstance(p, Point)
-                else mercantile.quadkey(mercantile.tile(p[1], p[0], zoom_level))
-            )
-            for p in points
-        }
+        quadkeys = set(cls.get_quadkeys_from_points(points, zoom_level))
         cls.logger.info(f"Generated {len(quadkeys)} unique quadkeys from points.")
         return cls(zoom_level=zoom_level, quadkeys=list(quadkeys), **kwargs)
 
@@ -219,6 +215,29 @@ class MercatorTiles(BaseModel):
             {"quadkey": self.quadkeys, "geometry": self.to_geoms()}, crs="EPSG:4326"
         )
 
+    @staticmethod
+    def get_quadkeys_from_points(
+        points: List[Union[Point, Tuple[float, float]]], zoom_level: int
+    ) -> List[str]:
+        """Get list of quadkeys for the provided points at specified zoom level.
+
+        Args:
+            points: List of points as either shapely Points or (lon, lat) tuples
+            zoom_level: Zoom level for the quadkeys
+
+        Returns:
+            List of quadkey strings
+        """
+        quadkeys = [
+            (
+                mercantile.quadkey(mercantile.tile(p.x, p.y, zoom_level))
+                if isinstance(p, Point)
+                else mercantile.quadkey(mercantile.tile(p[1], p[0], zoom_level))
+            )
+            for p in points
+        ]
+        return quadkeys
+
     def save(self, file: Union[str, Path], format: str = "json") -> None:
         """Save MercatorTiles to file in specified format."""
         with self.data_store.open(str(file), "wb" if format == "parquet" else "w") as f:
@@ -270,6 +289,10 @@ class CountryMercatorTiles(MercatorTiles):
             country=pycountry.countries.lookup(country).alpha_3,
         )
 
+        cls.logger.info(
+            f"Initializing Mercator zones for country: {country} at zoom level {zoom_level}"
+        )
+
         country_geom = (
             AdminBoundaries.create(
                 country_code=country,

gigaspatial/handlers/boundaries.py

@@ -10,7 +10,7 @@ import pycountry
 from gigaspatial.core.io.data_store import DataStore
 from gigaspatial.core.io.readers import read_dataset
 from gigaspatial.handlers.hdx import HDXConfig
-from gigaspatial.config import config
+from gigaspatial.config import config as global_config
 
 
 class AdminBoundary(BaseModel):
@@ -33,7 +33,6 @@ class AdminBoundary(BaseModel):
     )
 
     class Config:
-        # extra = "allow"
         arbitrary_types_allowed = True
 
 
@@ -48,7 +47,7 @@ class AdminBoundaries(BaseModel):
         description="Administrative level (e.g., 0=country, 1=state, etc.)",
     )
 
-    logger: ClassVar = config.get_logger("AdminBoundaries")
+    logger: ClassVar = global_config.get_logger("AdminBoundaries")
 
     _schema_config: ClassVar[Dict[str, Dict[str, str]]] = {
         "gadm": {
@@ -301,28 +300,50 @@ class AdminBoundaries(BaseModel):
         path: Optional[Union[str, "Path"]] = None,
         **kwargs,
     ) -> "AdminBoundaries":
-        """Factory method to create AdminBoundaries instance from either GADM or data store.
+        """
+        Factory method to create an AdminBoundaries instance using various data sources,
+        depending on the provided parameters and global configuration.
+
+        Loading Logic:
+            1. If a `data_store` is provided and either a `path` is given or
+               `global_config.ADMIN_BOUNDARIES_DATA_DIR` is set:
+                - If `path` is not provided but `country_code` is, the path is constructed
+                  using `global_config.get_admin_path()`.
+                - Loads boundaries from the specified data store and path.
+
+            2. If only `country_code` is provided (no data_store):
+                - Attempts to load boundaries from GeoRepo (if available).
+                - If GeoRepo is unavailable, attempts to load from GADM.
+                - If GADM fails, falls back to geoBoundaries.
+                - Raises an error if all sources fail.
+
+            3. If neither `country_code` nor `data_store` is provided:
+                - Raises a ValueError.
 
         Args:
-            country_code: ISO country code (2 or 3 letter) or country name
-            admin_level: Administrative level (0=country, 1=state/province, etc.)
-            data_store: Optional data store instance for loading from existing data
-            path: Optional path to data file (used with data_store)
-            **kwargs: Additional arguments passed to the underlying creation methods
+            country_code (Optional[str]): ISO country code (2 or 3 letter) or country name.
+            admin_level (int): Administrative level (0=country, 1=state/province, etc.).
+            data_store (Optional[DataStore]): Optional data store instance for loading from existing data.
+            path (Optional[Union[str, Path]]): Optional path to data file (used with data_store).
+            **kwargs: Additional arguments passed to the underlying creation methods.
 
         Returns:
-            AdminBoundaries: Configured instance
+            AdminBoundaries: Configured instance.
 
         Raises:
             ValueError: If neither country_code nor (data_store, path) are provided,
-                    or if country_code lookup fails
+                        or if country_code lookup fails.
+            RuntimeError: If all data sources fail to load boundaries.
 
-        Example:
-            # From country code
-            boundaries = AdminBoundaries.create(country_code="USA", admin_level=1)
+        Examples:
+            # Load from a data store (path auto-generated if not provided)
+            boundaries = AdminBoundaries.create(country_code="USA", admin_level=1, data_store=store)
 
-            # From data store
+            # Load from a specific file in a data store
             boundaries = AdminBoundaries.create(data_store=store, path="data.shp")
+
+            # Load from online sources (GeoRepo, GADM, geoBoundaries)
+            boundaries = AdminBoundaries.create(country_code="USA", admin_level=1)
         """
         cls.logger.info(
             f"Creating AdminBoundaries instance. Country: {country_code}, "
@@ -330,17 +351,21 @@ class AdminBoundaries(BaseModel):
             f"path provided: {path is not None}"
         )
 
+        from_data_store = data_store is not None and (
+            global_config.ADMIN_BOUNDARIES_DATA_DIR is not None or path is not None
+        )
+
         # Validate input parameters
         if not country_code and not data_store:
             raise ValueError("Either country_code or data_store must be provided.")
 
-        if data_store and not path and not country_code:
+        if from_data_store and not path and not country_code:
             raise ValueError(
                 "If data_store is provided, either path or country_code must also be specified."
             )
 
         # Handle data store path first
-        if data_store is not None:
+        if from_data_store:
             iso3_code = None
             if country_code:
                 try:
@@ -350,7 +375,7 @@ class AdminBoundaries(BaseModel):
 
             # Generate path if not provided
             if path is None and iso3_code:
-                path = config.get_admin_path(
+                path = global_config.get_admin_path(
                     country_code=iso3_code,
                     admin_level=admin_level,
                 )

gigaspatial/handlers/ghsl.py

@@ -14,7 +14,6 @@ import requests
 from tqdm import tqdm
 import zipfile
 import tempfile
-import shutil
 from pydantic import (
     HttpUrl,
     Field,
@@ -25,8 +24,6 @@ from pydantic import (
 import logging
 
 from gigaspatial.core.io.data_store import DataStore
-from gigaspatial.core.io.local_data_store import LocalDataStore
-from gigaspatial.handlers.boundaries import AdminBoundaries
 from gigaspatial.processing.tif_processor import TifProcessor
 from gigaspatial.handlers.base import (
     BaseHandlerConfig,
@@ -241,8 +238,8 @@ class GHSLDataConfig(BaseHandlerConfig):
             ValueError: If the input `source` is not one of the supported types.
         """
         if isinstance(source, gpd.GeoDataFrame):
-            # if source.crs != "EPSG:4326":
-            #    source = source.to_crs("EPSG:4326")
+            if source.crs != crs:
+                source = source.to_crs(crs)
             search_geom = source.geometry.unary_union
         elif isinstance(
             source,
@@ -273,7 +270,9 @@ class GHSLDataConfig(BaseHandlerConfig):
             tile_geom.intersects(search_geom) for tile_geom in self.tiles_gdf.geometry
         )
 
-        return self.tiles_gdf.loc[mask, "tile_id"].to_list()
+        intersecting_tiles = self.tiles_gdf.loc[mask, "tile_id"].to_list()
+
+        return intersecting_tiles
 
     def _get_product_info(self) -> dict:
         """Generate and return common product information used in multiple methods."""
@@ -340,7 +339,7 @@ class GHSLDataDownloader(BaseHandlerDownloader):
 
         Args:
             tile_id: tile ID to process.
-            extract: If True and the downloaded file is a zip, extract its contents. Defaults to False.
+            extract: If True and the downloaded file is a zip, extract its contents. Defaults to True.
             file_pattern: Optional regex pattern to filter extracted files (if extract=True).
             **kwargs: Additional parameters passed to download methods
 
@@ -356,14 +355,34 @@ class GHSLDataDownloader(BaseHandlerDownloader):
             return self._download_file(url, output_path)
 
         extracted_files: List[Path] = []
+        temp_downloaded_path: Optional[Path] = None
 
         try:
             with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as temp_file:
-                downloaded_path = self._download_file(url, Path(temp_file.name))
-                if not downloaded_path:
-                    return None
+                temp_downloaded_path = Path(temp_file.name)
+                self.logger.debug(
+                    f"Downloading {url} to temporary file: {temp_downloaded_path}"
+                )
+
+                response = requests.get(url, stream=True)
+                response.raise_for_status()
+
+                total_size = int(response.headers.get("content-length", 0))
+
+                with tqdm(
+                    total=total_size,
+                    unit="B",
+                    unit_scale=True,
+                    desc=f"Downloading {tile_id}",
+                ) as pbar:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        if chunk:
+                            temp_file.write(chunk)
+                            pbar.update(len(chunk))
+
+            self.logger.info(f"Successfully downloaded temporary file!")
 
-            with zipfile.ZipFile(str(downloaded_path), "r") as zip_ref:
+            with zipfile.ZipFile(str(temp_downloaded_path), "r") as zip_ref:
                 if file_pattern:
                     import re
 
@@ -385,9 +404,24 @@ class GHSLDataDownloader(BaseHandlerDownloader):
             Path(temp_file.name).unlink()
             return extracted_files
 
+        except requests.exceptions.RequestException as e:
+            self.logger.error(f"Failed to download {url} to temporary file: {e}")
+            return None
+        except zipfile.BadZipFile:
+            self.logger.error(f"Downloaded file for {tile_id} is not a valid zip file.")
+            return None
         except Exception as e:
             self.logger.error(f"Error downloading/extracting tile {tile_id}: {e}")
             return None
+        finally:
+            if temp_downloaded_path and temp_downloaded_path.exists():
+                try:
+                    temp_downloaded_path.unlink()
+                    self.logger.debug(f"Deleted temporary file: {temp_downloaded_path}")
+                except OSError as e:
+                    self.logger.warning(
+                        f"Could not delete temporary file {temp_downloaded_path}: {e}"
+                    )
 
     def download_data_units(
         self,
@@ -401,7 +435,7 @@ class GHSLDataDownloader(BaseHandlerDownloader):
 
         Args:
             tile_ids: A list of tile IDs to download.
-            extract: If True and the downloaded files are zips, extract their contents. Defaults to False.
+            extract: If True and the downloaded files are zips, extract their contents. Defaults to True.
             file_pattern: Optional regex pattern to filter extracted files (if extract=True).
             **kwargs: Additional parameters passed to download methods
 
@@ -456,7 +490,7 @@ class GHSLDataDownloader(BaseHandlerDownloader):
                       - A list of (latitude, longitude) tuples or Shapely Point objects.
                       - A Shapely BaseGeometry object (e.g., Polygon, MultiPolygon).
                       - A GeoDataFrame with geometry column in EPSG:4326.
-            extract: If True and the downloaded files are zips, extract their contents. Defaults to False.
+            extract: If True and the downloaded files are zips, extract their contents. Defaults to True.
             file_pattern: Optional regex pattern to filter extracted files (if extract=True).
             **kwargs: Additional keyword arguments. These will be passed down to
                       `AdminBoundaries.create()` (if `source` is a country)
@@ -496,7 +530,7 @@ class GHSLDataDownloader(BaseHandlerDownloader):
             country_geom_path: Optional path to a GeoJSON file containing the
                                country boundary. If provided, this boundary is used
                                instead of the default from `AdminBoundaries`.
-            extract: If True and the downloaded files are zips, extract their contents. Defaults to False.
+            extract: If True and the downloaded files are zips, extract their contents. Defaults to True.
             file_pattern: Optional regex pattern to filter extracted files (if extract=True).
             **kwargs: Additional keyword arguments that are passed to
                       `download_data_units`. For example, `extract` to download and extract.
@@ -770,3 +804,34 @@ class GHSLDataHandler(BaseHandler):
         return pd.concat(
             [tp.to_dataframe() for tp in tif_processors], ignore_index=True
         )
+    
+    def load_into_geodataframe(
+        self,
+        source: Union[
+            str,  # country
+            List[Union[tuple, Point]],  # points
+            BaseGeometry,  # geometry
+            gpd.GeoDataFrame,  # geodataframe
+            Path,  # path
+            List[Union[str, Path]],  # list of paths
+        ],
+        ensure_available: bool = True,
+        **kwargs,
+    ) -> pd.DataFrame:
+        """
+        Load GHSL data into a geopandas GeoDataFrame.
+
+        Args:
+            source: The data source specification
+            ensure_available: If True, ensure data is downloaded before loading
+            **kwargs: Additional parameters passed to load methods
+
+        Returns:
+            GeoDataFrame containing the GHSL data
+        """
+        tif_processors = self.load_data(
+            source=source, ensure_available=ensure_available, **kwargs
+        )
+        return pd.concat(
+            [tp.to_geodataframe() for tp in tif_processors], ignore_index=True
+        )

gigaspatial/handlers/rwi.py

@@ -2,6 +2,7 @@ import logging
 from typing import List, Optional, Union, Literal
 from pydantic.dataclasses import dataclass
 from datetime import datetime
+import pycountry
 
 from hdx.data.resource import Resource
 
@@ -36,8 +37,10 @@ class RWIConfig(HDXConfig):
         self, country: str, **kwargs
     ) -> List[Resource]:
         """Get relevant data units for a country, optionally filtering for latest version"""
-        resources = super().get_relevant_data_units_by_country(
-            country=country, key="url"
+        country = pycountry.countries.lookup(country)
+        values = [country.alpha_3]
+        resources = self.get_dataset_resources(
+            filter={"url": values},
         )
 
         if self.latest_only and len(resources) > 1: