giga-spatial 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl

gigaspatial/generators/zonal/mercator.py

@@ -15,25 +15,48 @@ from gigaspatial.generators.zonal.geometry import GeometryBasedZonalViewGenerato
 
 
 class MercatorViewGenerator(GeometryBasedZonalViewGenerator[T]):
-    """Mid-level class for zonal view generation based on geometries with identifiers.
+    """
+    Generates zonal views using Mercator tiles as the zones.
 
-    This class serves as an intermediate between the abstract ZonalViewGenerator and specific
-    implementations like MercatorViewGenerator or H3ViewGenerator. It handles the common case
-    where zones are defined by a mapping between zone identifiers and geometries, either
-    provided as a dictionary or as a GeoDataFrame.
+    This class specializes in creating zonal views where the zones are defined by
+    Mercator tiles. It extends the `GeometryBasedZonalViewGenerator` and leverages
+    the `MercatorTiles` and `CountryMercatorTiles` classes to generate tiles based on
+    various input sources.
 
-    The class extends the base functionality with methods for mapping common geospatial
-    datasets including GHSL (Global Human Settlement Layer), Google Open Buildings,
-    and Microsoft Global Buildings data.
+    The primary input source defines the geographical area of interest. This can be
+    a country, a specific geometry, a set of points, or even a list of predefined
+    quadkeys. The `zoom_level` determines the granularity of the Mercator tiles.
 
     Attributes:
-        zone_dict (Dict[T, Polygon]): Mapping of zone identifiers to geometries.
-        zone_id_column (str): Name of the column containing zone identifiers.
-        zone_data_crs (str): Coordinate reference system of the zone data.
-        _zone_gdf (gpd.GeoDataFrame): Cached GeoDataFrame representation of zones.
-        data_store (DataStore): For accessing input data.
-        generator_config (ZonalViewGeneratorConfig): Configuration for view generation.
-        logger: Logger instance for this class.
+        source (Union[str, BaseGeometry, gpd.GeoDataFrame, List[Union[Point, Tuple[float, float]]], List[str]]):
+            Specifies the geographic area or specific tiles to use. Can be:
+            - A country name (str): Uses `CountryMercatorTiles` to generate tiles covering the country.
+            - A Shapely geometry (BaseGeometry):  Uses `MercatorTiles.from_spatial` to create tiles intersecting the geometry.
+            - A GeoDataFrame (gpd.GeoDataFrame): Uses `MercatorTiles.from_spatial` to create tiles intersecting the geometries.
+            - A list of points (List[Union[Point, Tuple[float, float]]]):  Uses `MercatorTiles.from_spatial` to create tiles containing the points.
+            - A list of quadkeys (List[str]): Uses `MercatorTiles.from_quadkeys` to use the specified tiles directly.
+        zoom_level (int): The zoom level of the Mercator tiles. Higher zoom levels result in smaller, more detailed tiles.
+        predicate (str):  The spatial predicate used when filtering tiles based on a spatial source (e.g., "intersects", "contains"). Defaults to "intersects".
+        config (Optional[ZonalViewGeneratorConfig]): Configuration for the zonal view generation process.
+        data_store (Optional[DataStore]):  A DataStore instance for accessing data.
+        logger (Optional[logging.Logger]):  A logger instance for logging.
+
+    Methods:
+        _init_zone_data(source, zoom_level, predicate):  Initializes the Mercator tile GeoDataFrame based on the input source.
+        # Inherits other methods from GeometryBasedZonalViewGenerator, such as:
+        # map_ghsl(), map_google_buildings(), map_ms_buildings(), aggregate_data(), save_view()
+
+    Example:
+        # Create a MercatorViewGenerator for tiles covering Germany at zoom level 6
+        generator = MercatorViewGenerator(source="Germany", zoom_level=6)
+
+        # Create a MercatorViewGenerator for tiles intersecting a specific polygon
+        polygon = ... # Define a Shapely Polygon
+        generator = MercatorViewGenerator(source=polygon, zoom_level=8)
+
+        # Create a MercatorViewGenerator from a list of quadkeys
+        quadkeys = ["0020023131023032", "0020023131023033"]
+        generator = MercatorViewGenerator(source=quadkeys, zoom_level=12)
     """
 
     def __init__(
@@ -53,16 +76,19 @@ class MercatorViewGenerator(GeometryBasedZonalViewGenerator[T]):
     ):
 
         super().__init__(
-            zone_data=self._init_zone_data(source, zoom_level, predicate),
+            zone_data=self._init_zone_data(source, zoom_level, predicate, data_store),
             zone_id_column="quadkey",
             config=config,
             data_store=data_store,
             logger=logger,
         )
+        self.logger.info(f"Initialized MercatorViewGenerator")
 
-    def _init_zone_data(self, source, zoom_level, predicate):
+    def _init_zone_data(self, source, zoom_level, predicate, data_store=None):
         if isinstance(source, str):
-            tiles = CountryMercatorTiles.create(country=source, zoom_level=zoom_level)
+            tiles = CountryMercatorTiles.create(
+                country=source, zoom_level=zoom_level, data_store=data_store
+            )
         elif isinstance(source, (BaseGeometry, Iterable)):
             if isinstance(source, Iterable) and all(
                 isinstance(qk, str) for qk in source
@@ -73,6 +99,11 @@ class MercatorViewGenerator(GeometryBasedZonalViewGenerator[T]):
                     source=source, zoom_level=zoom_level, predicate=predicate
                 )
         else:
-            raise ValueError("sadadasfasfkasmf")
+            raise TypeError(
+                f"Unsupported source type for MercatorViewGenerator. 'source' must be "
+                f"a country name (str), a Shapely geometry, a GeoDataFrame, "
+                f"a list of quadkeys (str), or a list of (lon, lat) tuples/Shapely Point objects. "
+                f"Received type: {type(source)}."
+            )
 
         return tiles.to_geodataframe()

gigaspatial/grid/__init__.py

@@ -1 +1 @@
-from gigaspatial.grid.mercator_tiles import *
+from gigaspatial.grid.mercator_tiles import MercatorTiles, CountryMercatorTiles

gigaspatial/grid/mercator_tiles.py

@@ -4,10 +4,10 @@ import mercantile
 from shapely.geometry import box
 from shapely.geometry.base import BaseGeometry
 from shapely.strtree import STRtree
-from shapely import MultiPolygon, Polygon, Point
+from shapely import Point
 import json
 from pathlib import Path
-from pydantic import BaseModel, Field, PrivateAttr
+from pydantic import BaseModel, Field
 from typing import List, Union, Iterable, Optional, Tuple, ClassVar
 import pycountry
 
@@ -31,6 +31,9 @@ class MercatorTiles(BaseModel):
         if not quadkeys:
             cls.logger.warning("No quadkeys provided to from_quadkeys.")
             return cls(zoom_level=0, quadkeys=[])
+        cls.logger.info(
+            f"Initializing MercatorTiles from {len(quadkeys)} provided quadkeys."
+        )
         return cls(zoom_level=len(quadkeys[0]), quadkeys=set(quadkeys))
 
     @classmethod
@@ -120,14 +123,7 @@ class MercatorTiles(BaseModel):
         cls.logger.info(
             f"Creating MercatorTiles from {len(points)} points at zoom level: {zoom_level}"
         )
-        quadkeys = {
-            (
-                mercantile.quadkey(mercantile.tile(p.x, p.y, zoom_level))
-                if isinstance(p, Point)
-                else mercantile.quadkey(mercantile.tile(p[1], p[0], zoom_level))
-            )
-            for p in points
-        }
+        quadkeys = set(cls.get_quadkeys_from_points(points, zoom_level))
         cls.logger.info(f"Generated {len(quadkeys)} unique quadkeys from points.")
         return cls(zoom_level=zoom_level, quadkeys=list(quadkeys), **kwargs)
 
@@ -219,6 +215,29 @@ class MercatorTiles(BaseModel):
             {"quadkey": self.quadkeys, "geometry": self.to_geoms()}, crs="EPSG:4326"
         )
 
+    @staticmethod
+    def get_quadkeys_from_points(
+        points: List[Union[Point, Tuple[float, float]]], zoom_level: int
+    ) -> List[str]:
+        """Get list of quadkeys for the provided points at specified zoom level.
+
+        Args:
+            points: List of points as either shapely Points or (lon, lat) tuples
+            zoom_level: Zoom level for the quadkeys
+
+        Returns:
+            List of quadkey strings
+        """
+        quadkeys = [
+            (
+                mercantile.quadkey(mercantile.tile(p.x, p.y, zoom_level))
+                if isinstance(p, Point)
+                else mercantile.quadkey(mercantile.tile(p[1], p[0], zoom_level))
+            )
+            for p in points
+        ]
+        return quadkeys
+
     def save(self, file: Union[str, Path], format: str = "json") -> None:
         """Save MercatorTiles to file in specified format."""
         with self.data_store.open(str(file), "wb" if format == "parquet" else "w") as f:
@@ -270,6 +289,10 @@ class CountryMercatorTiles(MercatorTiles):
             country=pycountry.countries.lookup(country).alpha_3,
         )
 
+        cls.logger.info(
+            f"Initializing Mercator zones for country: {country} at zoom level {zoom_level}"
+        )
+
         country_geom = (
             AdminBoundaries.create(
                 country_code=country,

gigaspatial/handlers/__init__.py

@@ -37,4 +37,8 @@ from gigaspatial.handlers.unicef_georepo import (
     GeoRepoClient,
     get_country_boundaries_by_iso3,
 )
-from gigaspatial.handlers.giga import GigaSchoolLocationFetcher
+from gigaspatial.handlers.giga import (
+    GigaSchoolLocationFetcher,
+    GigaSchoolProfileFetcher,
+    GigaSchoolMeasurementsFetcher,
+)

gigaspatial/handlers/boundaries.py

@@ -4,11 +4,13 @@ import geopandas as gpd
 from pathlib import Path
 from urllib.error import HTTPError
 from shapely.geometry import Polygon, MultiPolygon, shape
+import tempfile
 import pycountry
 
 from gigaspatial.core.io.data_store import DataStore
 from gigaspatial.core.io.readers import read_dataset
-from gigaspatial.config import config
+from gigaspatial.handlers.hdx import HDXConfig
+from gigaspatial.config import config as global_config
 
 
 class AdminBoundary(BaseModel):
@@ -31,7 +33,6 @@ class AdminBoundary(BaseModel):
     )
 
     class Config:
-        # extra = "allow"
         arbitrary_types_allowed = True
 
 
@@ -46,7 +47,7 @@ class AdminBoundaries(BaseModel):
         description="Administrative level (e.g., 0=country, 1=state, etc.)",
     )
 
-    logger: ClassVar = config.get_logger("AdminBoundaries")
+    logger: ClassVar = global_config.get_logger("AdminBoundaries")
 
     _schema_config: ClassVar[Dict[str, Dict[str, str]]] = {
         "gadm": {
@@ -61,8 +62,31 @@ class AdminBoundaries(BaseModel):
             "name_en": "name_en",
             "country_code": "iso_3166_1_alpha_3",
         },
+        "geoBoundaries": {
+            "id": "shapeID",
+            "name": "shapeName",
+            "country_code": "shapeGroup",
+        },
     }
 
+    def to_geodataframe(self) -> gpd.GeoDataFrame:
+        """Convert the AdminBoundaries to a GeoDataFrame."""
+        if not self.boundaries:
+            if hasattr(self, "_empty_schema"):
+                columns = self._empty_schema
+            else:
+                columns = ["id", "name", "country_code", "geometry"]
+                if self.level > 0:
+                    columns.append("parent_id")
+
+            return gpd.GeoDataFrame(columns=columns, geometry="geometry", crs=4326)
+
+        return gpd.GeoDataFrame(
+            [boundary.model_dump() for boundary in self.boundaries],
+            geometry="geometry",
+            crs=4326,
+        )
+
     @classmethod
     def get_schema_config(cls) -> Dict[str, Dict[str, str]]:
         """Return field mappings for different data sources"""
@@ -100,6 +124,7 @@ class AdminBoundaries(BaseModel):
             cls.logger.warning(
                 f"Error loading GADM data for {country_code} at admin level {admin_level}: {str(e)}"
             )
+            cls.logger.info("Falling back to empty instance")
             return cls._create_empty_instance(country_code, admin_level, "gadm")
 
     @classmethod
@@ -138,6 +163,7 @@ class AdminBoundaries(BaseModel):
             cls.logger.warning(
                 f"No data found at {path} for admin level {admin_level}: {str(e)}"
             )
+            cls.logger.info("Falling back to empty instance")
             return cls._create_empty_instance(None, admin_level, "internal")
 
     @classmethod
@@ -202,6 +228,69 @@ class AdminBoundaries(BaseModel):
 
         return cls(boundaries=boundaries, level=admin_level)
 
+    @classmethod
+    def from_geoboundaries(cls, country_code, admin_level: int = 0):
+        cls.logger.info(
+            f"Searching for geoBoundaries data for country: {country_code}, admin level: {admin_level}"
+        )
+
+        country_datasets = HDXConfig.search_datasets(
+            query=f'dataseries_name:"geoBoundaries - Subnational Administrative Boundaries" AND groups:"{country_code.lower()}"',
+            rows=1,
+        )
+        if not country_datasets:
+            cls.logger.error(f"No datasets found for country: {country_code}")
+            raise ValueError(
+                "No resources found for the specified country. Please check your search parameters and try again."
+            )
+
+        cls.logger.info(f"Found dataset: {country_datasets[0].get('title', 'Unknown')}")
+
+        resources = [
+            resource
+            for resource in country_datasets[0].get_resources()
+            if (
+                resource.data["name"]
+                == f"geoBoundaries-{country_code.upper()}-ADM{admin_level}.geojson"
+            )
+        ]
+
+        if not resources:
+            cls.logger.error(
+                f"No resources found for {country_code} at admin level {admin_level}"
+            )
+            raise ValueError(
+                "No resources found for the specified criteria. Please check your search parameters and try again."
+            )
+
+        cls.logger.info(f"Found resource: {resources[0].data.get('name', 'Unknown')}")
+
+        try:
+            cls.logger.info("Downloading and processing boundary data...")
+            with tempfile.TemporaryDirectory() as tmpdir:
+                url, local_path = resources[0].download(folder=tmpdir)
+                cls.logger.debug(f"Downloaded file to temporary path: {local_path}")
+                with open(local_path, "rb") as f:
+                    gdf = gpd.read_file(f)
+
+            gdf = cls._map_fields(gdf, "geoBoundaries", admin_level)
+            boundaries = [
+                AdminBoundary(**row_dict) for row_dict in gdf.to_dict("records")
+            ]
+            cls.logger.info(
+                f"Successfully created {len(boundaries)} AdminBoundary objects"
+            )
+            return cls(boundaries=boundaries, level=admin_level)
+
+        except (ValueError, HTTPError, FileNotFoundError) as e:
+            cls.logger.warning(
+                f"Error loading geoBoundaries data for {country_code} at admin level {admin_level}: {str(e)}"
+            )
+            cls.logger.info("Falling back to empty instance")
+            return cls._create_empty_instance(
+                country_code, admin_level, "geoBoundaries"
+            )
+
     @classmethod
     def create(
         cls,
@@ -211,45 +300,152 @@ class AdminBoundaries(BaseModel):
         path: Optional[Union[str, "Path"]] = None,
         **kwargs,
     ) -> "AdminBoundaries":
-        """Factory method to create AdminBoundaries instance from either GADM or data store."""
+        """
+        Factory method to create an AdminBoundaries instance using various data sources,
+        depending on the provided parameters and global configuration.
+
+        Loading Logic:
+            1. If a `data_store` is provided and either a `path` is given or
+               `global_config.ADMIN_BOUNDARIES_DATA_DIR` is set:
+                - If `path` is not provided but `country_code` is, the path is constructed
+                  using `global_config.get_admin_path()`.
+                - Loads boundaries from the specified data store and path.
+
+            2. If only `country_code` is provided (no data_store):
+                - Attempts to load boundaries from GeoRepo (if available).
+                - If GeoRepo is unavailable, attempts to load from GADM.
+                - If GADM fails, falls back to geoBoundaries.
+                - Raises an error if all sources fail.
+
+            3. If neither `country_code` nor `data_store` is provided:
+                - Raises a ValueError.
+
+        Args:
+            country_code (Optional[str]): ISO country code (2 or 3 letter) or country name.
+            admin_level (int): Administrative level (0=country, 1=state/province, etc.).
+            data_store (Optional[DataStore]): Optional data store instance for loading from existing data.
+            path (Optional[Union[str, Path]]): Optional path to data file (used with data_store).
+            **kwargs: Additional arguments passed to the underlying creation methods.
+
+        Returns:
+            AdminBoundaries: Configured instance.
+
+        Raises:
+            ValueError: If neither country_code nor (data_store, path) are provided,
+                        or if country_code lookup fails.
+            RuntimeError: If all data sources fail to load boundaries.
+
+        Examples:
+            # Load from a data store (path auto-generated if not provided)
+            boundaries = AdminBoundaries.create(country_code="USA", admin_level=1, data_store=store)
+
+            # Load from a specific file in a data store
+            boundaries = AdminBoundaries.create(data_store=store, path="data.shp")
+
+            # Load from online sources (GeoRepo, GADM, geoBoundaries)
+            boundaries = AdminBoundaries.create(country_code="USA", admin_level=1)
+        """
         cls.logger.info(
-            f"Creating AdminBoundaries instance. Country: {country_code}, admin level: {admin_level}, data_store provided: {data_store is not None}, path provided: {path is not None}"
+            f"Creating AdminBoundaries instance. Country: {country_code}, "
+            f"admin level: {admin_level}, data_store provided: {data_store is not None}, "
+            f"path provided: {path is not None}"
         )
-        iso3_code = pycountry.countries.lookup(country_code).alpha_3
-        if data_store is not None:
-            if path is None:
-                if country_code is None:
-                    ValueError(
-                        "If data_store is provided, path or country_code must also be specified."
-                    )
-                path = config.get_admin_path(
+
+        from_data_store = data_store is not None and (
+            global_config.ADMIN_BOUNDARIES_DATA_DIR is not None or path is not None
+        )
+
+        # Validate input parameters
+        if not country_code and not data_store:
+            raise ValueError("Either country_code or data_store must be provided.")
+
+        if from_data_store and not path and not country_code:
+            raise ValueError(
+                "If data_store is provided, either path or country_code must also be specified."
+            )
+
+        # Handle data store path first
+        if from_data_store:
+            iso3_code = None
+            if country_code:
+                try:
+                    iso3_code = pycountry.countries.lookup(country_code).alpha_3
+                except LookupError as e:
+                    raise ValueError(f"Invalid country code '{country_code}': {e}")
+
+            # Generate path if not provided
+            if path is None and iso3_code:
+                path = global_config.get_admin_path(
                     country_code=iso3_code,
                     admin_level=admin_level,
                 )
+
             return cls.from_data_store(data_store, path, admin_level, **kwargs)
-        elif country_code is not None:
-            from gigaspatial.handlers.unicef_georepo import GeoRepoClient
 
+        # Handle country code path
+        if country_code is not None:
             try:
-                client = GeoRepoClient()
-                if client.check_connection():
-                    cls.logger.info("GeoRepo connection successful.")
-                    return cls.from_georepo(
-                        iso3_code,
-                        admin_level=admin_level,
-                    )
-            except ValueError as e:
+                iso3_code = pycountry.countries.lookup(country_code).alpha_3
+            except LookupError as e:
+                raise ValueError(f"Invalid country code '{country_code}': {e}")
+
+            # Try GeoRepo first
+            if cls._try_georepo(iso3_code, admin_level):
+                return cls.from_georepo(iso3_code, admin_level=admin_level)
+
+            # Fallback to GADM
+            try:
+                cls.logger.info("Attempting to load from GADM.")
+                return cls.from_gadm(iso3_code, admin_level, **kwargs)
+            except Exception as e:
                 cls.logger.warning(
-                    f"GeoRepo initialization failed: {str(e)}. Falling back to GADM."
+                    f"GADM loading failed: {e}. Falling back to geoBoundaries."
                 )
+
+            # Final fallback to geoBoundaries
+            try:
+                return cls.from_geoboundaries(iso3_code, admin_level)
             except Exception as e:
-                cls.logger.warning(f"GeoRepo error: {str(e)}. Falling back to GADM.")
+                cls.logger.error(f"All data sources failed. geoBoundaries error: {e}")
+                raise RuntimeError(
+                    f"Failed to load administrative boundaries for {country_code} "
+                    f"from all available sources (GeoRepo, GADM, geoBoundaries)."
+                ) from e
 
-            return cls.from_gadm(iso3_code, admin_level, **kwargs)
-        else:
-            raise ValueError(
-                "Either country_code or (data_store, path) must be provided."
-            )
+        # This should never be reached due to validation above
+        raise ValueError("Unexpected error: no valid data source could be determined.")
+
+    @classmethod
+    def _try_georepo(cls, iso3_code: str, admin_level: int) -> bool:
+        """Helper method to test GeoRepo availability.
+
+        Args:
+            iso3_code: ISO3 country code
+            admin_level: Administrative level
+
+        Returns:
+            bool: True if GeoRepo is available and working, False otherwise
+        """
+        try:
+            from gigaspatial.handlers.unicef_georepo import GeoRepoClient
+
+            client = GeoRepoClient()
+            if client.check_connection():
+                cls.logger.info("GeoRepo connection successful.")
+                return True
+            else:
+                cls.logger.info("GeoRepo connection failed.")
+                return False
+
+        except ImportError:
+            cls.logger.info("GeoRepo client not available (import failed).")
+            return False
+        except ValueError as e:
+            cls.logger.warning(f"GeoRepo initialization failed: {e}")
+            return False
+        except Exception as e:
+            cls.logger.warning(f"GeoRepo error: {e}")
+            return False
 
     @classmethod
     def _create_empty_instance(
@@ -288,21 +484,3 @@ class AdminBoundaries(BaseModel):
                 field_mapping[v] = k
 
         return gdf.rename(columns=field_mapping)
-
-    def to_geodataframe(self) -> gpd.GeoDataFrame:
-        """Convert the AdminBoundaries to a GeoDataFrame."""
-        if not self.boundaries:
-            if hasattr(self, "_empty_schema"):
-                columns = self._empty_schema
-            else:
-                columns = ["id", "name", "country_code", "geometry"]
-                if self.level > 0:
-                    columns.append("parent_id")
-
-            return gpd.GeoDataFrame(columns=columns, geometry="geometry", crs=4326)
-
-        return gpd.GeoDataFrame(
-            [boundary.model_dump() for boundary in self.boundaries],
-            geometry="geometry",
-            crs=4326,
-        )