giga-spatial 0.6.4__py3-none-any.whl → 0.6.6__py3-none-any.whl

gigaspatial/handlers/boundaries.py

@@ -10,7 +10,7 @@ import pycountry
 from gigaspatial.core.io.data_store import DataStore
 from gigaspatial.core.io.readers import read_dataset
 from gigaspatial.handlers.hdx import HDXConfig
-from gigaspatial.config import config
+from gigaspatial.config import config as global_config
 
 
 class AdminBoundary(BaseModel):
@@ -33,7 +33,6 @@ class AdminBoundary(BaseModel):
     )
 
     class Config:
-        # extra = "allow"
         arbitrary_types_allowed = True
 
 
@@ -48,7 +47,7 @@ class AdminBoundaries(BaseModel):
         description="Administrative level (e.g., 0=country, 1=state, etc.)",
     )
 
-    logger: ClassVar = config.get_logger("AdminBoundaries")
+    logger: ClassVar = global_config.get_logger("AdminBoundaries")
 
     _schema_config: ClassVar[Dict[str, Dict[str, str]]] = {
         "gadm": {
@@ -292,6 +291,56 @@ class AdminBoundaries(BaseModel):
                 country_code, admin_level, "geoBoundaries"
             )
 
+    @classmethod
+    def from_global_country_boundaries(cls, scale: str = "medium") -> "AdminBoundaries":
+        """
+        Load global country boundaries from Natural Earth Data.
+
+        Args:
+            scale (str): One of 'large', 'medium', 'small'.
+                - 'large'  -> 10m
+                - 'medium' -> 50m
+                - 'small'  -> 110m
+        Returns:
+            AdminBoundaries: All country boundaries at admin_level=0
+        """
+        scale_map = {
+            "large": "10m",
+            "medium": "50m",
+            "small": "110m",
+        }
+        if scale not in scale_map:
+            raise ValueError(
+                f"Invalid scale '{scale}'. Choose from 'large', 'medium', 'small'."
+            )
+        scale_folder = scale_map[scale]
+        url = f"https://naciscdn.org/naturalearth/{scale_folder}/cultural/ne_{scale_folder}_admin_0_countries.zip"
+        cls.logger.info(f"Loading Natural Earth global country boundaries from {url}")
+        try:
+            gdf = gpd.read_file(url)
+            # Map fields to AdminBoundary schema
+            boundaries = []
+            for _, row in gdf.iterrows():
+                iso_a3 = row.get("ISO_A3_EH") or row.get("ISO_A3") or row.get("ADM0_A3")
+                name = row.get("NAME") or row.get("ADMIN") or row.get("SOVEREIGNT")
+                geometry = row.get("geometry")
+                if not iso_a3 or not name or geometry is None:
+                    continue
+                boundary = AdminBoundary(
+                    id=iso_a3,
+                    name=name,
+                    geometry=geometry,
+                    country_code=iso_a3,
+                )
+                boundaries.append(boundary)
+            cls.logger.info(
+                f"Loaded {len(boundaries)} country boundaries from Natural Earth."
+            )
+            return cls(boundaries=boundaries, level=0)
+        except Exception as e:
+            cls.logger.error(f"Failed to load Natural Earth global boundaries: {e}")
+            raise
+
     @classmethod
     def create(
         cls,
@@ -301,28 +350,50 @@ class AdminBoundaries(BaseModel):
         path: Optional[Union[str, "Path"]] = None,
         **kwargs,
     ) -> "AdminBoundaries":
-        """Factory method to create AdminBoundaries instance from either GADM or data store.
+        """
+        Factory method to create an AdminBoundaries instance using various data sources,
+        depending on the provided parameters and global configuration.
+
+        Loading Logic:
+            1. If a `data_store` is provided and either a `path` is given or
+               `global_config.ADMIN_BOUNDARIES_DATA_DIR` is set:
+                - If `path` is not provided but `country_code` is, the path is constructed
+                  using `global_config.get_admin_path()`.
+                - Loads boundaries from the specified data store and path.
+
+            2. If only `country_code` is provided (no data_store):
+                - Attempts to load boundaries from GeoRepo (if available).
+                - If GeoRepo is unavailable, attempts to load from GADM.
+                - If GADM fails, falls back to geoBoundaries.
+                - Raises an error if all sources fail.
+
+            3. If neither `country_code` nor `data_store` is provided:
+                - Raises a ValueError.
 
         Args:
-            country_code: ISO country code (2 or 3 letter) or country name
-            admin_level: Administrative level (0=country, 1=state/province, etc.)
-            data_store: Optional data store instance for loading from existing data
-            path: Optional path to data file (used with data_store)
-            **kwargs: Additional arguments passed to the underlying creation methods
+            country_code (Optional[str]): ISO country code (2 or 3 letter) or country name.
+            admin_level (int): Administrative level (0=country, 1=state/province, etc.).
+            data_store (Optional[DataStore]): Optional data store instance for loading from existing data.
+            path (Optional[Union[str, Path]]): Optional path to data file (used with data_store).
+            **kwargs: Additional arguments passed to the underlying creation methods.
 
         Returns:
-            AdminBoundaries: Configured instance
+            AdminBoundaries: Configured instance.
 
         Raises:
             ValueError: If neither country_code nor (data_store, path) are provided,
-                    or if country_code lookup fails
+                        or if country_code lookup fails.
+            RuntimeError: If all data sources fail to load boundaries.
 
-        Example:
-            # From country code
-            boundaries = AdminBoundaries.create(country_code="USA", admin_level=1)
+        Examples:
+            # Load from a data store (path auto-generated if not provided)
+            boundaries = AdminBoundaries.create(country_code="USA", admin_level=1, data_store=store)
 
-            # From data store
+            # Load from a specific file in a data store
             boundaries = AdminBoundaries.create(data_store=store, path="data.shp")
+
+            # Load from online sources (GeoRepo, GADM, geoBoundaries)
+            boundaries = AdminBoundaries.create(country_code="USA", admin_level=1)
         """
         cls.logger.info(
             f"Creating AdminBoundaries instance. Country: {country_code}, "
@@ -330,17 +401,21 @@ class AdminBoundaries(BaseModel):
             f"path provided: {path is not None}"
         )
 
+        from_data_store = data_store is not None and (
+            global_config.ADMIN_BOUNDARIES_DATA_DIR is not None or path is not None
+        )
+
         # Validate input parameters
         if not country_code and not data_store:
             raise ValueError("Either country_code or data_store must be provided.")
 
-        if data_store and not path and not country_code:
+        if from_data_store and not path and not country_code:
             raise ValueError(
                 "If data_store is provided, either path or country_code must also be specified."
             )
 
         # Handle data store path first
-        if data_store is not None:
+        if from_data_store:
             iso3_code = None
             if country_code:
                 try:
@@ -350,7 +425,7 @@ class AdminBoundaries(BaseModel):
 
             # Generate path if not provided
             if path is None and iso3_code:
-                path = config.get_admin_path(
+                path = global_config.get_admin_path(
                     country_code=iso3_code,
                     admin_level=admin_level,
                 )

gigaspatial/handlers/ghsl.py

@@ -14,7 +14,6 @@ import requests
 from tqdm import tqdm
 import zipfile
 import tempfile
-import shutil
 from pydantic import (
     HttpUrl,
     Field,
@@ -25,8 +24,6 @@ from pydantic import (
 import logging
 
 from gigaspatial.core.io.data_store import DataStore
-from gigaspatial.core.io.local_data_store import LocalDataStore
-from gigaspatial.handlers.boundaries import AdminBoundaries
 from gigaspatial.processing.tif_processor import TifProcessor
 from gigaspatial.handlers.base import (
     BaseHandlerConfig,
@@ -241,8 +238,8 @@ class GHSLDataConfig(BaseHandlerConfig):
             ValueError: If the input `source` is not one of the supported types.
         """
         if isinstance(source, gpd.GeoDataFrame):
-            # if source.crs != "EPSG:4326":
-            #    source = source.to_crs("EPSG:4326")
+            if source.crs != crs:
+                source = source.to_crs(crs)
             search_geom = source.geometry.unary_union
         elif isinstance(
             source,
@@ -273,7 +270,9 @@ class GHSLDataConfig(BaseHandlerConfig):
             tile_geom.intersects(search_geom) for tile_geom in self.tiles_gdf.geometry
         )
 
-        return self.tiles_gdf.loc[mask, "tile_id"].to_list()
+        intersecting_tiles = self.tiles_gdf.loc[mask, "tile_id"].to_list()
+
+        return intersecting_tiles
 
     def _get_product_info(self) -> dict:
         """Generate and return common product information used in multiple methods."""
@@ -340,7 +339,7 @@ class GHSLDataDownloader(BaseHandlerDownloader):
 
         Args:
             tile_id: tile ID to process.
-            extract: If True and the downloaded file is a zip, extract its contents. Defaults to False.
+            extract: If True and the downloaded file is a zip, extract its contents. Defaults to True.
             file_pattern: Optional regex pattern to filter extracted files (if extract=True).
             **kwargs: Additional parameters passed to download methods
 
@@ -356,14 +355,34 @@ class GHSLDataDownloader(BaseHandlerDownloader):
             return self._download_file(url, output_path)
 
         extracted_files: List[Path] = []
+        temp_downloaded_path: Optional[Path] = None
 
         try:
             with tempfile.NamedTemporaryFile(delete=False, suffix=".zip") as temp_file:
-                downloaded_path = self._download_file(url, Path(temp_file.name))
-                if not downloaded_path:
-                    return None
+                temp_downloaded_path = Path(temp_file.name)
+                self.logger.debug(
+                    f"Downloading {url} to temporary file: {temp_downloaded_path}"
+                )
+
+                response = requests.get(url, stream=True)
+                response.raise_for_status()
+
+                total_size = int(response.headers.get("content-length", 0))
+
+                with tqdm(
+                    total=total_size,
+                    unit="B",
+                    unit_scale=True,
+                    desc=f"Downloading {tile_id}",
+                ) as pbar:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        if chunk:
+                            temp_file.write(chunk)
+                            pbar.update(len(chunk))
+
+            self.logger.info(f"Successfully downloaded temporary file!")
 
-            with zipfile.ZipFile(str(downloaded_path), "r") as zip_ref:
+            with zipfile.ZipFile(str(temp_downloaded_path), "r") as zip_ref:
                 if file_pattern:
                     import re
 
@@ -385,9 +404,24 @@ class GHSLDataDownloader(BaseHandlerDownloader):
             Path(temp_file.name).unlink()
             return extracted_files
 
+        except requests.exceptions.RequestException as e:
+            self.logger.error(f"Failed to download {url} to temporary file: {e}")
+            return None
+        except zipfile.BadZipFile:
+            self.logger.error(f"Downloaded file for {tile_id} is not a valid zip file.")
+            return None
         except Exception as e:
             self.logger.error(f"Error downloading/extracting tile {tile_id}: {e}")
             return None
+        finally:
+            if temp_downloaded_path and temp_downloaded_path.exists():
+                try:
+                    temp_downloaded_path.unlink()
+                    self.logger.debug(f"Deleted temporary file: {temp_downloaded_path}")
+                except OSError as e:
+                    self.logger.warning(
+                        f"Could not delete temporary file {temp_downloaded_path}: {e}"
+                    )
 
     def download_data_units(
         self,
@@ -401,7 +435,7 @@ class GHSLDataDownloader(BaseHandlerDownloader):
 
         Args:
             tile_ids: A list of tile IDs to download.
-            extract: If True and the downloaded files are zips, extract their contents. Defaults to False.
+            extract: If True and the downloaded files are zips, extract their contents. Defaults to True.
             file_pattern: Optional regex pattern to filter extracted files (if extract=True).
             **kwargs: Additional parameters passed to download methods
 
@@ -456,7 +490,7 @@ class GHSLDataDownloader(BaseHandlerDownloader):
                       - A list of (latitude, longitude) tuples or Shapely Point objects.
                       - A Shapely BaseGeometry object (e.g., Polygon, MultiPolygon).
                       - A GeoDataFrame with geometry column in EPSG:4326.
-            extract: If True and the downloaded files are zips, extract their contents. Defaults to False.
+            extract: If True and the downloaded files are zips, extract their contents. Defaults to True.
             file_pattern: Optional regex pattern to filter extracted files (if extract=True).
             **kwargs: Additional keyword arguments. These will be passed down to
                       `AdminBoundaries.create()` (if `source` is a country)
@@ -496,7 +530,7 @@ class GHSLDataDownloader(BaseHandlerDownloader):
             country_geom_path: Optional path to a GeoJSON file containing the
                                country boundary. If provided, this boundary is used
                                instead of the default from `AdminBoundaries`.
-            extract: If True and the downloaded files are zips, extract their contents. Defaults to False.
+            extract: If True and the downloaded files are zips, extract their contents. Defaults to True.
             file_pattern: Optional regex pattern to filter extracted files (if extract=True).
             **kwargs: Additional keyword arguments that are passed to
                       `download_data_units`. For example, `extract` to download and extract.
@@ -563,7 +597,7 @@ class GHSLDataReader(BaseHandlerReader):
         logger: Optional[logging.Logger] = None,
     ):
         """
-        Initialize the downloader.
+        Initialize the reader.
 
         Args:
             config: Configuration for the GHSL dataset, either as a GHSLDataConfig object or a dictionary of parameters
@@ -770,3 +804,46 @@ class GHSLDataHandler(BaseHandler):
         return pd.concat(
             [tp.to_dataframe() for tp in tif_processors], ignore_index=True
         )
+
+    def load_into_geodataframe(
+        self,
+        source: Union[
+            str,  # country
+            List[Union[tuple, Point]],  # points
+            BaseGeometry,  # geometry
+            gpd.GeoDataFrame,  # geodataframe
+            Path,  # path
+            List[Union[str, Path]],  # list of paths
+        ],
+        ensure_available: bool = True,
+        **kwargs,
+    ) -> gpd.GeoDataFrame:
+        """
+        Load GHSL data into a geopandas GeoDataFrame.
+
+        Args:
+            source: The data source specification
+            ensure_available: If True, ensure data is downloaded before loading
+            **kwargs: Additional parameters passed to load methods
+
+        Returns:
+            GeoDataFrame containing the GHSL data
+        """
+        tif_processors = self.load_data(
+            source=source, ensure_available=ensure_available, **kwargs
+        )
+        return pd.concat(
+            [tp.to_geodataframe() for tp in tif_processors], ignore_index=True
+        )
+
+    def get_available_data_info(
+        self,
+        source: Union[
+            str,  # country
+            List[Union[tuple, Point]],  # points
+            BaseGeometry,  # geometry
+            gpd.GeoDataFrame,  # geodataframe
+        ],
+        **kwargs,
+    ) -> dict:
+        return super().get_available_data_info(source, file_ext=".tif", **kwargs)

gigaspatial/handlers/rwi.py

@@ -2,6 +2,7 @@ import logging
 from typing import List, Optional, Union, Literal
 from pydantic.dataclasses import dataclass
 from datetime import datetime
+import pycountry
 
 from hdx.data.resource import Resource
 
@@ -36,8 +37,10 @@ class RWIConfig(HDXConfig):
         self, country: str, **kwargs
     ) -> List[Resource]:
         """Get relevant data units for a country, optionally filtering for latest version"""
-        resources = super().get_relevant_data_units_by_country(
-            country=country, key="url"
+        country = pycountry.countries.lookup(country)
+        values = [country.alpha_3]
+        resources = self.get_dataset_resources(
+            filter={"url": values},
         )
 
         if self.latest_only and len(resources) > 1: