giga-spatial 0.6.3__py3-none-any.whl → 0.6.4__py3-none-any.whl

{giga_spatial-0.6.3.dist-info → giga_spatial-0.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: giga-spatial
-Version: 0.6.3
+Version: 0.6.4
 Summary: A package for spatial data download & processing
 Home-page: https://github.com/unicef/giga-spatial
 Author: Utku Can Ozturk

{giga_spatial-0.6.3.dist-info → giga_spatial-0.6.4.dist-info}/RECORD

@@ -1,6 +1,6 @@
-giga_spatial-0.6.3.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-gigaspatial/__init__.py,sha256=zYiFHqR7JwbvdK9dvKrh-RTNfUqjHUwC4CTcFAPVYLc,22
-gigaspatial/config.py,sha256=yMf1ofOU0_I6iKDqshiFSYmK6TDIVpPm1AZo4e2okHU,8166
+giga_spatial-0.6.4.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+gigaspatial/__init__.py,sha256=WMmvm2Keb76yMz8OL_h4fKT34Xpi-1BVfCiTn2QGzz4,22
+gigaspatial/config.py,sha256=PR6n6NDDD4560zWEbaFiYSitr9PAKik915cxCCMZNQc,8392
 gigaspatial/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 gigaspatial/core/io/__init__.py,sha256=y4QNWx6US1-adTuAO_NZwLmjzSQj25HNDL5hUGvEHZc,263
 gigaspatial/core/io/adls_data_store.py,sha256=Zv-D_8d_2h57HnCUTJb0JWWjXqR_0XH4F8Nu_UFZK9E,11975
@@ -19,15 +19,15 @@ gigaspatial/generators/zonal/geometry.py,sha256=XPcX5lT7X7Z1vn72sN-VKLb2hDP9F_w3
 gigaspatial/generators/zonal/mercator.py,sha256=R_KlaqF4lnc0cRqVfcNVO8i0Re21_6w7pnclVKSohcY,3125
 gigaspatial/grid/__init__.py,sha256=H8SnNAMDafJXJ9bUp2zU0Z3t6s8niqY5rGP5nFhnbLA,45
 gigaspatial/grid/mercator_tiles.py,sha256=Z_3M4sy1tyxywAo2wmBb6niBP3x-IWgwMkmUp8LOSDg,10492
-gigaspatial/handlers/__init__.py,sha256=pqK3rJtelOAkBaWNhpGy2t-p_zrwO-9BqABLQufTXF0,1449
+gigaspatial/handlers/__init__.py,sha256=R2rugXR5kF4lLkSO1fjpVDYK_jWdD8U2NbXbW71Ezv8,1523
 gigaspatial/handlers/base.py,sha256=rL94c3wDjsqzLp4na8FfYXW6tNjVGX6v4M-Ce4LrAro,26413
-gigaspatial/handlers/boundaries.py,sha256=hoO-b5MlFYwlCWogApcFyEx6OnxMJG29lqJurNGwOWg,11260
+gigaspatial/handlers/boundaries.py,sha256=UM0lFcTzy64ADdMnPOkzLGJ-OG5P7KyoZtA91GTWxYs,17242
 gigaspatial/handlers/ghsl.py,sha256=GHao8lkmj1C0-QFqNwH9jr0Lqzu6NTj_7ooQdj1h6ok,27760
-gigaspatial/handlers/giga.py,sha256=2aP1EenDAQXn-h-uCyuVxEVZvAFEvrL17_z0MiS8FDs,4867
+gigaspatial/handlers/giga.py,sha256=F5ZfcE37a24X-c6Xhyt72C9eZZbyN_gV7w_InxKFMQQ,28348
 gigaspatial/handlers/google_open_buildings.py,sha256=Liqk7qJhDtB4Ia4uhBe44LFcf-XVKBjRfj-pWlE5erY,16594
-gigaspatial/handlers/hdx.py,sha256=DNw-LhxuJU3eNGihQGyPJT0a1PaOCupNHr7BDGal4Zo,18088
+gigaspatial/handlers/hdx.py,sha256=LTEs_xZF1yPhD8dAdZ_YN8Vcan7iB5_tZ8NjF_ip6u0,18001
 gigaspatial/handlers/mapbox_image.py,sha256=M_nkJ_b1PD8FG1ajVgSycCb0NRTAI_SLpHdzszNetKA,7786
-gigaspatial/handlers/maxar_image.py,sha256=g5YVGV-8JjeG9bGBOp7ZfKani22J4izXX4hnB9A99Jk,10272
+gigaspatial/handlers/maxar_image.py,sha256=kcc8uGljQB0Yh0MKBA7lT7KwBbNZwFzuyBklR3db1P4,10204
 gigaspatial/handlers/microsoft_global_buildings.py,sha256=bQ5WHIv3v0wWrZZUbZkKPRjgdlqIxlK7CV_0zSvdrTw,20292
 gigaspatial/handlers/ookla_speedtest.py,sha256=EcvSAxJZ9GPfzYnT_C85Qgy2ecc9ndf70Pklk53OdC8,6506
 gigaspatial/handlers/opencellid.py,sha256=KuJqd-5-RO5ZzyDaBSrTgCK2ib5N_m3RUcPlX5heWwI,10683
@@ -41,7 +41,7 @@ gigaspatial/processing/geo.py,sha256=D-S3IlhQwLIxrCcxy6NhNmKLrOIjoRHfK_eZJGKpe2U
 gigaspatial/processing/sat_images.py,sha256=YUbH5MFNzl6NX49Obk14WaFcr1s3SyGJIOk-kRpbBNg,1429
 gigaspatial/processing/tif_processor.py,sha256=zqcP_ioo9KHNJ6H0uba4UghW4MToTRwq1iE-nZbb8zA,21101
 gigaspatial/processing/utils.py,sha256=HC85vGKQakxlkoQAkZmeAXWHsenAwTIRn7jPKUA7x20,1500
-giga_spatial-0.6.3.dist-info/METADATA,sha256=Aw5adPdTcA3AuJBmZgAG4rJQYW4dJqw2GT90mYE7cgU,7467
-giga_spatial-0.6.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-giga_spatial-0.6.3.dist-info/top_level.txt,sha256=LZsccgw6H4zXT7m6Y4XChm-Y5LjHAwZ2hkGN_B3ExmI,12
-giga_spatial-0.6.3.dist-info/RECORD,,
+giga_spatial-0.6.4.dist-info/METADATA,sha256=WQUWSdjlmfh09kkX20cgudrGHWmldXlNbh4DNjB0Xgo,7467
+giga_spatial-0.6.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+giga_spatial-0.6.4.dist-info/top_level.txt,sha256=LZsccgw6H4zXT7m6Y4XChm-Y5LjHAwZ2hkGN_B3ExmI,12
+giga_spatial-0.6.4.dist-info/RECORD,,

gigaspatial/__init__.py

@@ -1 +1 @@
-__version__ = "0.6.3"
+__version__ = "0.6.4"

gigaspatial/config.py

@@ -32,6 +32,12 @@ class Config(BaseSettings):
     GIGA_SCHOOL_LOCATION_API_KEY: str = Field(
         default="", alias="GIGA_SCHOOL_LOCATION_API_KEY"
     )
+    GIGA_SCHOOL_PROFILE_API_KEY: str = Field(
+        default="", alias="GIGA_SCHOOL_PROFILE_API_KEY"
+    )
+    GIGA_SCHOOL_MEASUREMENTS_API_KEY: str = Field(
+        default="", alias="GIGA_SCHOOL_MEASUREMENTS_API_KEY"
+    )
 
     ROOT_DATA_DIR: Path = Field(
         default=Path("."),

gigaspatial/handlers/__init__.py

@@ -37,4 +37,8 @@ from gigaspatial.handlers.unicef_georepo import (
     GeoRepoClient,
     get_country_boundaries_by_iso3,
 )
-from gigaspatial.handlers.giga import GigaSchoolLocationFetcher
+from gigaspatial.handlers.giga import (
+    GigaSchoolLocationFetcher,
+    GigaSchoolProfileFetcher,
+    GigaSchoolMeasurementsFetcher,
+)

gigaspatial/handlers/boundaries.py

@@ -4,10 +4,12 @@ import geopandas as gpd
 from pathlib import Path
 from urllib.error import HTTPError
 from shapely.geometry import Polygon, MultiPolygon, shape
+import tempfile
 import pycountry
 
 from gigaspatial.core.io.data_store import DataStore
 from gigaspatial.core.io.readers import read_dataset
+from gigaspatial.handlers.hdx import HDXConfig
 from gigaspatial.config import config
 
 
@@ -61,8 +63,31 @@ class AdminBoundaries(BaseModel):
             "name_en": "name_en",
             "country_code": "iso_3166_1_alpha_3",
         },
+        "geoBoundaries": {
+            "id": "shapeID",
+            "name": "shapeName",
+            "country_code": "shapeGroup",
+        },
     }
 
+    def to_geodataframe(self) -> gpd.GeoDataFrame:
+        """Convert the AdminBoundaries to a GeoDataFrame."""
+        if not self.boundaries:
+            if hasattr(self, "_empty_schema"):
+                columns = self._empty_schema
+            else:
+                columns = ["id", "name", "country_code", "geometry"]
+                if self.level > 0:
+                    columns.append("parent_id")
+
+            return gpd.GeoDataFrame(columns=columns, geometry="geometry", crs=4326)
+
+        return gpd.GeoDataFrame(
+            [boundary.model_dump() for boundary in self.boundaries],
+            geometry="geometry",
+            crs=4326,
+        )
+
     @classmethod
     def get_schema_config(cls) -> Dict[str, Dict[str, str]]:
         """Return field mappings for different data sources"""
@@ -100,6 +125,7 @@ class AdminBoundaries(BaseModel):
             cls.logger.warning(
                 f"Error loading GADM data for {country_code} at admin level {admin_level}: {str(e)}"
             )
+            cls.logger.info("Falling back to empty instance")
             return cls._create_empty_instance(country_code, admin_level, "gadm")
 
     @classmethod
@@ -138,6 +164,7 @@ class AdminBoundaries(BaseModel):
             cls.logger.warning(
                 f"No data found at {path} for admin level {admin_level}: {str(e)}"
             )
+            cls.logger.info("Falling back to empty instance")
             return cls._create_empty_instance(None, admin_level, "internal")
 
     @classmethod
@@ -202,6 +229,69 @@ class AdminBoundaries(BaseModel):
 
         return cls(boundaries=boundaries, level=admin_level)
 
+    @classmethod
+    def from_geoboundaries(cls, country_code, admin_level: int = 0):
+        cls.logger.info(
+            f"Searching for geoBoundaries data for country: {country_code}, admin level: {admin_level}"
+        )
+
+        country_datasets = HDXConfig.search_datasets(
+            query=f'dataseries_name:"geoBoundaries - Subnational Administrative Boundaries" AND groups:"{country_code.lower()}"',
+            rows=1,
+        )
+        if not country_datasets:
+            cls.logger.error(f"No datasets found for country: {country_code}")
+            raise ValueError(
+                "No resources found for the specified country. Please check your search parameters and try again."
+            )
+
+        cls.logger.info(f"Found dataset: {country_datasets[0].get('title', 'Unknown')}")
+
+        resources = [
+            resource
+            for resource in country_datasets[0].get_resources()
+            if (
+                resource.data["name"]
+                == f"geoBoundaries-{country_code.upper()}-ADM{admin_level}.geojson"
+            )
+        ]
+
+        if not resources:
+            cls.logger.error(
+                f"No resources found for {country_code} at admin level {admin_level}"
+            )
+            raise ValueError(
+                "No resources found for the specified criteria. Please check your search parameters and try again."
+            )
+
+        cls.logger.info(f"Found resource: {resources[0].data.get('name', 'Unknown')}")
+
+        try:
+            cls.logger.info("Downloading and processing boundary data...")
+            with tempfile.TemporaryDirectory() as tmpdir:
+                url, local_path = resources[0].download(folder=tmpdir)
+                cls.logger.debug(f"Downloaded file to temporary path: {local_path}")
+                with open(local_path, "rb") as f:
+                    gdf = gpd.read_file(f)
+
+            gdf = cls._map_fields(gdf, "geoBoundaries", admin_level)
+            boundaries = [
+                AdminBoundary(**row_dict) for row_dict in gdf.to_dict("records")
+            ]
+            cls.logger.info(
+                f"Successfully created {len(boundaries)} AdminBoundary objects"
+            )
+            return cls(boundaries=boundaries, level=admin_level)
+
+        except (ValueError, HTTPError, FileNotFoundError) as e:
+            cls.logger.warning(
+                f"Error loading geoBoundaries data for {country_code} at admin level {admin_level}: {str(e)}"
+            )
+            cls.logger.info("Falling back to empty instance")
+            return cls._create_empty_instance(
+                country_code, admin_level, "geoBoundaries"
+            )
+
     @classmethod
     def create(
         cls,
@@ -211,45 +301,126 @@ class AdminBoundaries(BaseModel):
         path: Optional[Union[str, "Path"]] = None,
         **kwargs,
     ) -> "AdminBoundaries":
-        """Factory method to create AdminBoundaries instance from either GADM or data store."""
+        """Factory method to create AdminBoundaries instance from either GADM or data store.
+
+        Args:
+            country_code: ISO country code (2 or 3 letter) or country name
+            admin_level: Administrative level (0=country, 1=state/province, etc.)
+            data_store: Optional data store instance for loading from existing data
+            path: Optional path to data file (used with data_store)
+            **kwargs: Additional arguments passed to the underlying creation methods
+
+        Returns:
+            AdminBoundaries: Configured instance
+
+        Raises:
+            ValueError: If neither country_code nor (data_store, path) are provided,
+                    or if country_code lookup fails
+
+        Example:
+            # From country code
+            boundaries = AdminBoundaries.create(country_code="USA", admin_level=1)
+
+            # From data store
+            boundaries = AdminBoundaries.create(data_store=store, path="data.shp")
+        """
         cls.logger.info(
-            f"Creating AdminBoundaries instance. Country: {country_code}, admin level: {admin_level}, data_store provided: {data_store is not None}, path provided: {path is not None}"
+            f"Creating AdminBoundaries instance. Country: {country_code}, "
+            f"admin level: {admin_level}, data_store provided: {data_store is not None}, "
+            f"path provided: {path is not None}"
         )
-        iso3_code = pycountry.countries.lookup(country_code).alpha_3
+
+        # Validate input parameters
+        if not country_code and not data_store:
+            raise ValueError("Either country_code or data_store must be provided.")
+
+        if data_store and not path and not country_code:
+            raise ValueError(
+                "If data_store is provided, either path or country_code must also be specified."
+            )
+
+        # Handle data store path first
         if data_store is not None:
-            if path is None:
-                if country_code is None:
-                    ValueError(
-                        "If data_store is provided, path or country_code must also be specified."
-                    )
+            iso3_code = None
+            if country_code:
+                try:
+                    iso3_code = pycountry.countries.lookup(country_code).alpha_3
+                except LookupError as e:
+                    raise ValueError(f"Invalid country code '{country_code}': {e}")
+
+            # Generate path if not provided
+            if path is None and iso3_code:
                 path = config.get_admin_path(
                     country_code=iso3_code,
                     admin_level=admin_level,
                 )
+
             return cls.from_data_store(data_store, path, admin_level, **kwargs)
-        elif country_code is not None:
-            from gigaspatial.handlers.unicef_georepo import GeoRepoClient
 
+        # Handle country code path
+        if country_code is not None:
             try:
-                client = GeoRepoClient()
-                if client.check_connection():
-                    cls.logger.info("GeoRepo connection successful.")
-                    return cls.from_georepo(
-                        iso3_code,
-                        admin_level=admin_level,
-                    )
-            except ValueError as e:
+                iso3_code = pycountry.countries.lookup(country_code).alpha_3
+            except LookupError as e:
+                raise ValueError(f"Invalid country code '{country_code}': {e}")
+
+            # Try GeoRepo first
+            if cls._try_georepo(iso3_code, admin_level):
+                return cls.from_georepo(iso3_code, admin_level=admin_level)
+
+            # Fallback to GADM
+            try:
+                cls.logger.info("Attempting to load from GADM.")
+                return cls.from_gadm(iso3_code, admin_level, **kwargs)
+            except Exception as e:
                 cls.logger.warning(
-                    f"GeoRepo initialization failed: {str(e)}. Falling back to GADM."
+                    f"GADM loading failed: {e}. Falling back to geoBoundaries."
                 )
+
+            # Final fallback to geoBoundaries
+            try:
+                return cls.from_geoboundaries(iso3_code, admin_level)
             except Exception as e:
-                cls.logger.warning(f"GeoRepo error: {str(e)}. Falling back to GADM.")
+                cls.logger.error(f"All data sources failed. geoBoundaries error: {e}")
+                raise RuntimeError(
+                    f"Failed to load administrative boundaries for {country_code} "
+                    f"from all available sources (GeoRepo, GADM, geoBoundaries)."
+                ) from e
 
-            return cls.from_gadm(iso3_code, admin_level, **kwargs)
-        else:
-            raise ValueError(
-                "Either country_code or (data_store, path) must be provided."
-            )
+        # This should never be reached due to validation above
+        raise ValueError("Unexpected error: no valid data source could be determined.")
+
+    @classmethod
+    def _try_georepo(cls, iso3_code: str, admin_level: int) -> bool:
+        """Helper method to test GeoRepo availability.
+
+        Args:
+            iso3_code: ISO3 country code
+            admin_level: Administrative level
+
+        Returns:
+            bool: True if GeoRepo is available and working, False otherwise
+        """
+        try:
+            from gigaspatial.handlers.unicef_georepo import GeoRepoClient
+
+            client = GeoRepoClient()
+            if client.check_connection():
+                cls.logger.info("GeoRepo connection successful.")
+                return True
+            else:
+                cls.logger.info("GeoRepo connection failed.")
+                return False
+
+        except ImportError:
+            cls.logger.info("GeoRepo client not available (import failed).")
+            return False
+        except ValueError as e:
+            cls.logger.warning(f"GeoRepo initialization failed: {e}")
+            return False
+        except Exception as e:
+            cls.logger.warning(f"GeoRepo error: {e}")
+            return False
 
     @classmethod
     def _create_empty_instance(
@@ -288,21 +459,3 @@ class AdminBoundaries(BaseModel):
                 field_mapping[v] = k
 
         return gdf.rename(columns=field_mapping)
-
-    def to_geodataframe(self) -> gpd.GeoDataFrame:
-        """Convert the AdminBoundaries to a GeoDataFrame."""
-        if not self.boundaries:
-            if hasattr(self, "_empty_schema"):
-                columns = self._empty_schema
-            else:
-                columns = ["id", "name", "country_code", "geometry"]
-                if self.level > 0:
-                    columns.append("parent_id")
-
-            return gpd.GeoDataFrame(columns=columns, geometry="geometry", crs=4326)
-
-        return gpd.GeoDataFrame(
-            [boundary.model_dump() for boundary in self.boundaries],
-            geometry="geometry",
-            crs=4326,
-        )

gigaspatial/handlers/giga.py

@@ -1,10 +1,12 @@
 import requests
 import pandas as pd
 import time
+from datetime import datetime, date
 from pydantic.dataclasses import dataclass, Field
 from pydantic import ConfigDict
 from shapely.geometry import Point
 import pycountry
+from typing import Optional, Union
 import logging
 
 from gigaspatial.config import config as global_config
@@ -143,3 +145,642 @@ class GigaSchoolLocationFetcher:
         self.logger.info(f"Created geometry for all {len(df)} records")
 
         return df
+
+
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
+class GigaSchoolProfileFetcher:
+    """
+    Fetch and process school profile data from the Giga School Profile API.
+    This includes connectivity information and other school details.
+    """
+
+    country: str = Field(...)
+    api_url: str = Field(
+        default="https://uni-ooi-giga-maps-service.azurewebsites.net/api/v1/schools_profile/",
+        description="Base URL for the Giga School Profile API",
+    )
+    api_key: str = global_config.GIGA_SCHOOL_PROFILE_API_KEY
+    page_size: int = Field(default=1000, description="Number of records per API page")
+    sleep_time: float = Field(
+        default=0.2, description="Sleep time between API requests"
+    )
+    giga_id_school: Optional[str] = Field(
+        default=None, description="Optional specific giga school ID to fetch"
+    )
+
+    logger: logging.Logger = Field(default=None, repr=False)
+
+    def __post_init__(self):
+        try:
+            self.country = pycountry.countries.lookup(self.country).alpha_3
+        except LookupError:
+            raise ValueError(f"Invalid country code provided: {self.country}")
+
+        if self.logger is None:
+            self.logger = global_config.get_logger(self.__class__.__name__)
+
+    def fetch_profiles(self, **kwargs) -> pd.DataFrame:
+        """
+        Fetch and process school profiles including connectivity information.
+
+        Args:
+            **kwargs: Additional parameters for customization
+                - page_size: Override default page size
+                - sleep_time: Override default sleep time between requests
+                - max_pages: Limit the number of pages to fetch
+                - giga_id_school: Override default giga_id_school filter
+
+        Returns:
+            pd.DataFrame: School profiles with connectivity and geospatial info.
+        """
+        # Override defaults with kwargs if provided
+        page_size = kwargs.get("page_size", self.page_size)
+        sleep_time = kwargs.get("sleep_time", self.sleep_time)
+        max_pages = kwargs.get("max_pages", None)
+        giga_id_school = kwargs.get("giga_id_school", self.giga_id_school)
+
+        # Prepare headers
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Accept": "application/json",
+        }
+
+        all_data = []
+        page = 1
+
+        self.logger.info(
+            f"Starting to fetch school profiles for country: {self.country}"
+        )
+
+        if giga_id_school:
+            self.logger.info(f"Filtering for specific school ID: {giga_id_school}")
+
+        while True:
+            # Check if we've reached max_pages limit
+            if max_pages and page > max_pages:
+                self.logger.info(f"Reached maximum pages limit: {max_pages}")
+                break
+
+            # Build parameters
+            params = {
+                "country_iso3_code": self.country,
+                "page": page,
+                "size": page_size,
+            }
+
+            # Add giga_id_school filter if specified
+            if giga_id_school:
+                params["giga_id_school"] = giga_id_school
+
+            try:
+                self.logger.debug(f"Fetching page {page} with params: {params}")
+                response = requests.get(self.api_url, headers=headers, params=params)
+                response.raise_for_status()
+
+                parsed = response.json()
+                data = parsed.get("data", [])
+
+            except requests.exceptions.RequestException as e:
+                self.logger.error(f"Request failed on page {page}: {e}")
+                break
+            except ValueError as e:
+                self.logger.error(f"Failed to parse JSON response on page {page}: {e}")
+                break
+
+            # Check if we got any data
+            if not data:
+                self.logger.info(f"No data on page {page}. Stopping.")
+                break
+
+            all_data.extend(data)
+            self.logger.info(f"Fetched page {page} with {len(data)} records")
+
+            # If we got fewer records than page_size, we've reached the end
+            if len(data) < page_size:
+                self.logger.info("Reached end of data (partial page received)")
+                break
+
+            # If filtering by specific school ID, we likely only need one page
+            if giga_id_school:
+                self.logger.info(
+                    "Specific school ID requested, stopping after first page"
+                )
+                break
+
+            page += 1
+
+            # Sleep to be respectful to the API
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+
+        self.logger.info(f"Finished fetching. Total records: {len(all_data)}")
+
+        # Convert to DataFrame and process
+        if not all_data:
+            self.logger.warning("No data fetched, returning empty DataFrame")
+            return pd.DataFrame()
+
+        df = pd.DataFrame(all_data)
+
+        return df
+
+    def get_connectivity_summary(self, df: pd.DataFrame) -> dict:
+        """
+        Generate a summary of connectivity statistics from the fetched data.
+
+        Args:
+            df: DataFrame with school profile data
+
+        Returns:
+            dict: Summary statistics about connectivity
+        """
+        if df.empty:
+            return {"error": "No data available"}
+
+        summary = {
+            "total_schools": len(df),
+            "country": (
+                df["country_iso3_code"].iloc[0]
+                if "country_iso3_code" in df.columns
+                else "Unknown"
+            ),
+        }
+
+        # Administrative region analysis
+        if "admin1" in df.columns:
+            admin1_counts = df["admin1"].value_counts().head(10).to_dict()
+            summary["top_admin1_regions"] = admin1_counts
+
+        if "admin2" in df.columns:
+            admin2_counts = df["admin2"].value_counts().head(10).to_dict()
+            summary["top_admin2_regions"] = admin2_counts
+
+        # Connectivity analysis
+        if "connectivity" in df.columns:
+            connected_count = df["connectivity"].sum()
+            summary["schools_with_connectivity"] = int(connected_count)
+            summary["connectivity_percentage"] = connected_count / len(df) * 100
+
+        if "connectivity_RT" in df.columns:
+            rt_connected_count = df["connectivity_RT"].sum()
+            summary["schools_with_realtime_connectivity"] = int(rt_connected_count)
+            summary["realtime_connectivity_percentage"] = (
+                rt_connected_count / len(df) * 100
+            )
+
+        # Connectivity type analysis
+        if "connectivity_type" in df.columns:
+
+            if not all(df.connectivity_type.isna()):
+                from collections import Counter
+
+                type_counts = dict(Counter(df.connectivity_type.dropna().to_list()))
+                summary["connectivity_types_breakdown"] = type_counts
+
+        # Data source analysis
+        if "connectivity_RT_datasource" in df.columns:
+            datasource_counts = (
+                df["connectivity_RT_datasource"].value_counts().to_dict()
+            )
+            summary["realtime_connectivity_datasources"] = datasource_counts
+
+        if "school_data_source" in df.columns:
+            school_datasource_counts = df["school_data_source"].value_counts().to_dict()
+            summary["school_data_sources"] = school_datasource_counts
+
+        self.logger.info("Generated connectivity summary")
+        return summary
+
+
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
+class GigaSchoolMeasurementsFetcher:
+    """
+    Fetch and process school daily realtime connectivity measurements from the Giga API.
+    This includes download/upload speeds, latency, and connectivity performance data.
+    """
+
+    country: str = Field(...)
+    start_date: Union[str, date, datetime] = Field(...)
+    end_date: Union[str, date, datetime] = Field(...)
+    api_url: str = Field(
+        default="https://uni-ooi-giga-maps-service.azurewebsites.net/api/v1/all_measurements",
+        description="Base URL for the Giga School Measurements API",
+    )
+    api_key: str = global_config.GIGA_SCHOOL_MEASUREMENTS_API_KEY
+    page_size: int = Field(default=1000, description="Number of records per API page")
+    sleep_time: float = Field(
+        default=0.2, description="Sleep time between API requests"
+    )
+    giga_id_school: Optional[str] = Field(
+        default=None, description="Optional specific giga school ID to fetch"
+    )
+
+    logger: logging.Logger = Field(default=None, repr=False)
+
+    def __post_init__(self):
+        try:
+            self.country = pycountry.countries.lookup(self.country).alpha_3
+        except LookupError:
+            raise ValueError(f"Invalid country code provided: {self.country}")
+
+        # Convert dates to string format if needed
+        self.start_date = self._format_date(self.start_date)
+        self.end_date = self._format_date(self.end_date)
+
+        # Validate date range
+        if self.start_date > self.end_date:
+            raise ValueError("start_date must be before or equal to end_date")
+
+        if self.logger is None:
+            self.logger = global_config.get_logger(self.__class__.__name__)
+
+    def _format_date(self, date_input: Union[str, date, datetime]) -> str:
+        """
+        Convert date input to string format expected by API (YYYY-MM-DD).
+
+        Args:
+            date_input: Date in various formats
+
+        Returns:
+            str: Date in YYYY-MM-DD format
+        """
+        if isinstance(date_input, str):
+            # Assume it's already in correct format or parse it
+            try:
+                parsed_date = datetime.strptime(date_input, "%Y-%m-%d")
+                return date_input
+            except ValueError:
+                try:
+                    parsed_date = pd.to_datetime(date_input)
+                    return parsed_date.strftime("%Y-%m-%d")
+                except:
+                    raise ValueError(
+                        f"Invalid date format: {date_input}. Expected YYYY-MM-DD"
+                    )
+        elif isinstance(date_input, (date, datetime)):
+            return date_input.strftime("%Y-%m-%d")
+        else:
+            raise ValueError(f"Invalid date type: {type(date_input)}")
+
+    def fetch_measurements(self, **kwargs) -> pd.DataFrame:
+        """
+        Fetch and process school connectivity measurements.
+
+        Args:
+            **kwargs: Additional parameters for customization
+                - page_size: Override default page size
+                - sleep_time: Override default sleep time between requests
+                - max_pages: Limit the number of pages to fetch
+                - giga_id_school: Override default giga_id_school filter
+                - start_date: Override default start_date
+                - end_date: Override default end_date
+
+        Returns:
+            pd.DataFrame: School measurements with connectivity performance data.
+        """
+        # Override defaults with kwargs if provided
+        page_size = kwargs.get("page_size", self.page_size)
+        sleep_time = kwargs.get("sleep_time", self.sleep_time)
+        max_pages = kwargs.get("max_pages", None)
+        giga_id_school = kwargs.get("giga_id_school", self.giga_id_school)
+        start_date = kwargs.get("start_date", self.start_date)
+        end_date = kwargs.get("end_date", self.end_date)
+
+        # Format dates if overridden
+        if start_date != self.start_date:
+            start_date = self._format_date(start_date)
+        if end_date != self.end_date:
+            end_date = self._format_date(end_date)
+
+        # Prepare headers
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Accept": "application/json",
+        }
+
+        all_data = []
+        page = 1
+
+        self.logger.info(
+            f"Starting to fetch measurements for country: {self.country} "
+            f"from {start_date} to {end_date}"
+        )
+
+        if giga_id_school:
+            self.logger.info(f"Filtering for specific school ID: {giga_id_school}")
+
+        while True:
+            # Check if we've reached max_pages limit
+            if max_pages and page > max_pages:
+                self.logger.info(f"Reached maximum pages limit: {max_pages}")
+                break
+
+            # Build parameters
+            params = {
+                "country_iso3_code": self.country,
+                "start_date": start_date,
+                "end_date": end_date,
+                "page": page,
+                "size": page_size,
+            }
+
+            # Add giga_id_school filter if specified
+            if giga_id_school:
+                params["giga_id_school"] = giga_id_school
+
+            try:
+                self.logger.debug(f"Fetching page {page} with params: {params}")
+                response = requests.get(self.api_url, headers=headers, params=params)
+                response.raise_for_status()
+
+                parsed = response.json()
+                data = parsed.get("data", [])
+
+            except requests.exceptions.RequestException as e:
+                self.logger.error(f"Request failed on page {page}: {e}")
+                break
+            except ValueError as e:
+                self.logger.error(f"Failed to parse JSON response on page {page}: {e}")
+                break
+
+            # Check if we got any data
+            if not data:
+                self.logger.info(f"No data on page {page}. Stopping.")
+                break
+
+            all_data.extend(data)
+            self.logger.info(f"Fetched page {page} with {len(data)} records")
+
+            # If we got fewer records than page_size, we've reached the end
+            if len(data) < page_size:
+                self.logger.info("Reached end of data (partial page received)")
+                break
+
+            # If filtering by specific school ID, we might only need one page
+            if giga_id_school and len(all_data) > 0:
+                self.logger.info(
+                    "Specific school ID requested, checking if more data needed"
+                )
+
+            page += 1
+
+            # Sleep to be respectful to the API
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+
+        self.logger.info(f"Finished fetching. Total records: {len(all_data)}")
+
+        # Convert to DataFrame and process
+        if not all_data:
+            self.logger.warning("No data fetched, returning empty DataFrame")
+            return pd.DataFrame()
+
+        df = pd.DataFrame(all_data)
+        df = self._process_measurements_data(df)
+
+        return df
+
+    def _process_measurements_data(self, df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Process and enhance the DataFrame with measurement performance metrics.
+
+        Args:
+            df: Raw DataFrame from API
+
+        Returns:
+            pd.DataFrame: Enhanced DataFrame with processed measurement data
+        """
+        if df.empty:
+            return df
+
+        # Convert date column to datetime
+        if "date" in df.columns:
+            df["date"] = pd.to_datetime(df["date"], errors="coerce")
+            df["date_only"] = df["date"].dt.date
+            df["year"] = df["date"].dt.year
+            df["month"] = df["date"].dt.month
+            df["day_of_week"] = df["date"].dt.day_name()
+            self.logger.info("Processed date fields")
+
+        # Process speed measurements
+        numeric_columns = ["download_speed", "upload_speed", "latency"]
+        for col in numeric_columns:
+            if col in df.columns:
+                df[col] = pd.to_numeric(df[col], errors="coerce")
+
+        # Create performance categories
+        if "download_speed" in df.columns:
+            df["download_speed_category"] = pd.cut(
+                df["download_speed"],
+                bins=[0, 5, 25, 100, float("inf")],
+                labels=[
+                    "Very Slow (<5 Mbps)",
+                    "Slow (5-25 Mbps)",
+                    "Moderate (25-100 Mbps)",
+                    "Fast (>100 Mbps)",
+                ],
+                include_lowest=True,
+            )
+
+        if "upload_speed" in df.columns:
+            df["upload_speed_category"] = pd.cut(
+                df["upload_speed"],
+                bins=[0, 1, 10, 50, float("inf")],
+                labels=[
+                    "Very Slow (<1 Mbps)",
+                    "Slow (1-10 Mbps)",
+                    "Moderate (10-50 Mbps)",
+                    "Fast (>50 Mbps)",
+                ],
+                include_lowest=True,
+            )
+
+        if "latency" in df.columns:
+            df["latency_category"] = pd.cut(
+                df["latency"],
+                bins=[0, 50, 150, 300, float("inf")],
+                labels=[
+                    "Excellent (<50ms)",
+                    "Good (50-150ms)",
+                    "Fair (150-300ms)",
+                    "Poor (>300ms)",
+                ],
+                include_lowest=True,
+            )
+
+        # Create quality flags
+        if "download_speed" in df.columns and "upload_speed" in df.columns:
+            df["has_broadband"] = (df["download_speed"] >= 25) & (
+                df["upload_speed"] >= 3
+            )
+            df["has_basic_connectivity"] = (df["download_speed"] >= 1) & (
+                df["upload_speed"] >= 0.5
+            )
+
+        # Flag measurements with missing data
+        df["has_complete_measurement"] = (
+            df["download_speed"].notna()
+            & df["upload_speed"].notna()
+            & df["latency"].notna()
+        )
+
+        self.logger.info(f"Processed measurement data for {len(df)} records")
+
+        return df
+
+    def get_performance_summary(self, df: pd.DataFrame) -> dict:
+        """
+        Generate a comprehensive summary of connectivity performance metrics.
+
+        Args:
+            df: DataFrame with measurement data
+
+        Returns:
+            dict: Summary statistics about connectivity performance
+        """
+        if df.empty:
+            return {"error": "No data available"}
+
+        summary = {
+            "total_measurements": len(df),
+            "country": (
+                df["country_iso3_code"].iloc[0]
+                if "country_iso3_code" in df.columns
+                else "Unknown"
+            ),
+            "date_range": {
+                "start": (
+                    df["date"].min().strftime("%Y-%m-%d")
+                    if "date" in df.columns
+                    else None
+                ),
+                "end": (
+                    df["date"].max().strftime("%Y-%m-%d")
+                    if "date" in df.columns
+                    else None
+                ),
+            },
+        }
+
+        # School coverage
+        if "giga_id_school" in df.columns:
+            unique_schools = df["giga_id_school"].nunique()
+            summary["unique_schools_measured"] = unique_schools
+            summary["avg_measurements_per_school"] = (
+                len(df) / unique_schools if unique_schools > 0 else 0
+            )
+
+        # Speed statistics
+        for speed_col in ["download_speed", "upload_speed"]:
+            if speed_col in df.columns:
+                speed_data = df[speed_col].dropna()
+                if len(speed_data) > 0:
+                    summary[f"{speed_col}_stats"] = {
+                        "mean": float(speed_data.mean()),
+                        "median": float(speed_data.median()),
+                        "min": float(speed_data.min()),
+                        "max": float(speed_data.max()),
+                        "std": float(speed_data.std()),
+                    }
+
+        # Latency statistics
+        if "latency" in df.columns:
+            latency_data = df["latency"].dropna()
+            if len(latency_data) > 0:
+                summary["latency_stats"] = {
+                    "mean": float(latency_data.mean()),
+                    "median": float(latency_data.median()),
+                    "min": float(latency_data.min()),
+                    "max": float(latency_data.max()),
+                    "std": float(latency_data.std()),
+                }
+
+        # Performance categories
+        for cat_col in [
+            "download_speed_category",
+            "upload_speed_category",
+            "latency_category",
+        ]:
+            if cat_col in df.columns:
+                cat_counts = df[cat_col].value_counts().to_dict()
+                summary[cat_col.replace("_category", "_breakdown")] = cat_counts
+
+        # Quality metrics
+        if "has_broadband" in df.columns:
+            summary["broadband_capable_measurements"] = int(df["has_broadband"].sum())
+            summary["broadband_percentage"] = float(df["has_broadband"].mean() * 100)
+
+        if "has_basic_connectivity" in df.columns:
+            summary["basic_connectivity_measurements"] = int(
+                df["has_basic_connectivity"].sum()
+            )
+            summary["basic_connectivity_percentage"] = float(
+                df["has_basic_connectivity"].mean() * 100
+            )
+
+        # Data completeness
+        if "has_complete_measurement" in df.columns:
+            summary["complete_measurements"] = int(df["has_complete_measurement"].sum())
+            summary["data_completeness_percentage"] = float(
+                df["has_complete_measurement"].mean() * 100
+            )
+
+        # Data sources
+        if "data_source" in df.columns:
+            source_counts = df["data_source"].value_counts().to_dict()
+            summary["data_sources"] = source_counts
+
+        # Temporal patterns
+        if "day_of_week" in df.columns:
+            day_counts = df["day_of_week"].value_counts().to_dict()
+            summary["measurements_by_day_of_week"] = day_counts
+
+        self.logger.info("Generated performance summary")
+        return summary
+
+    def get_school_performance_comparison(
+        self, df: pd.DataFrame, top_n: int = 10
+    ) -> dict:
+        """
+        Compare performance across schools.
+
+        Args:
+            df: DataFrame with measurement data
+            top_n: Number of top/bottom schools to include
+
+        Returns:
+            dict: School performance comparison
+        """
+        if df.empty or "giga_id_school" not in df.columns:
+            return {"error": "No school data available"}
+
+        school_stats = (
+            df.groupby("giga_id_school")
+            .agg(
+                {
+                    "download_speed": ["mean", "median", "count"],
+                    "upload_speed": ["mean", "median"],
+                    "latency": ["mean", "median"],
+                    "has_broadband": (
+                        "mean" if "has_broadband" in df.columns else lambda x: None
+                    ),
+                }
+            )
+            .round(2)
+        )
+
+        # Flatten column names
+        school_stats.columns = ["_".join(col).strip() for col in school_stats.columns]
+
+        # Sort by download speed
+        if "download_speed_mean" in school_stats.columns:
+            top_schools = school_stats.nlargest(top_n, "download_speed_mean")
+            bottom_schools = school_stats.nsmallest(top_n, "download_speed_mean")
+
+            return {
+                "top_performing_schools": top_schools.to_dict("index"),
+                "bottom_performing_schools": bottom_schools.to_dict("index"),
+                "total_schools_analyzed": len(school_stats),
+            }
+
+        return {"error": "Insufficient data for school comparison"}

gigaspatial/handlers/hdx.py

@@ -1,13 +1,9 @@
-import os
 import logging
 from tqdm import tqdm
 from pathlib import Path
-from typing import List, Optional, Tuple, Union, Dict, Any, Iterable
+from typing import List, Optional, Union, Dict, Any, Iterable
 import tempfile
-import functools
-import multiprocessing
 
-import pandas as pd
 import geopandas as gpd
 from pydantic import Field, ConfigDict
 from pydantic.dataclasses import dataclass
@@ -50,6 +46,48 @@ class HDXConfig(BaseHandlerConfig):
     _hdx_configured: bool = Field(default=False, init=False)
     dataset: Optional[Dataset] = Field(default=None, init=False)
 
+    @staticmethod
+    def search_datasets(
+        query: str,
+        rows: int = None,
+        sort: str = "relevance asc, metadata_modified desc",
+        hdx_site: str = "prod",
+        user_agent: str = "gigaspatial",
+    ) -> List[Dict]:
+        """Search for datasets in HDX before initializing the class.
+
+        Args:
+            query: Search query string
+            rows: Number of results per page. Defaults to all datasets (sys.maxsize).
+            sort: Sort order - one of 'relevance', 'views_recent', 'views_total', 'last_modified' (default: 'relevance')
+            hdx_site: HDX site to use - 'prod' or 'test' (default: 'prod')
+            user_agent: User agent for HDX API requests (default: 'gigaspatial')
+
+        Returns:
+            List of dataset dictionaries containing search results
+
+        Example:
+            >>> results = HDXConfig.search_datasets("population", rows=5)
+            >>> for dataset in results:
+            >>>     print(f"Name: {dataset['name']}, Title: {dataset['title']}")
+        """
+        try:
+            Configuration.create(
+                hdx_site=hdx_site,
+                user_agent=user_agent,
+                hdx_read_only=True,
+            )
+        except:
+            pass
+
+        try:
+            results = Dataset.search_in_hdx(query=query, rows=rows, sort=sort)
+
+            return results
+        except Exception as e:
+            logging.error(f"Error searching HDX datasets: {str(e)}")
+            raise
+
     def __post_init__(self):
         super().__post_init__()
         try:
@@ -85,7 +123,11 @@ class HDXConfig(BaseHandlerConfig):
             self.logger.info(f"Fetching HDX dataset: {self.dataset_name}")
             dataset = Dataset.read_from_hdx(self.dataset_name)
             if not dataset:
-                raise ValueError(f"Dataset '{self.dataset_name}' not found on HDX")
+                raise ValueError(
+                    f"Dataset '{self.dataset_name}' not found on HDX. "
+                    "Please verify the dataset name or use search_datasets() "
+                    "to find available datasets."
+                )
             return dataset
         except Exception as e:
             self.logger.error(f"Error fetching HDX dataset: {str(e)}")
@@ -386,9 +428,9 @@ class HDXReader(BaseHandlerReader):
         self, source_data_path: List[Union[str, Path]], **kwargs
     ) -> Any:
         """Load data from paths"""
-        if len(source_data_path)==1:
+        if len(source_data_path) == 1:
             return read_dataset(self.data_store, source_data_path[0])
-        
+
         all_data = {}
         for file_path in source_data_path:
             try:
@@ -401,49 +443,6 @@ class HDXReader(BaseHandlerReader):
         resources = self.config.list_resources()
         return self.load_from_paths(resources)
 
-    # def read_resource(
-    #     self, resource_file: str
-    # ) -> Union[pd.DataFrame, gpd.GeoDataFrame]:
-    #     """Read a specific resource file from the dataset using the data_store."""
-    #     if not self.dataset_path:
-    #         raise ValueError("No dataset path configured")
-
-    #     file_path = str(self.dataset_path / resource_file)
-
-    #     if not self.data_store.file_exists(file_path):
-    #         raise FileNotFoundError(
-    #             f"Resource file {resource_file} not found in dataset"
-    #         )
-
-    #     try:
-    #         return read_dataset(self.data_store, file_path)
-    #     except Exception as e:
-    #         raise ValueError(f"Could not read file {file_path}: {str(e)}")
-
-    # def read_all_resources(self) -> Dict[str, Union[pd.DataFrame, gpd.GeoDataFrame]]:
-    #     """Read all resources in the dataset directory using the data_store."""
-    #     resources = self.list_resources()
-    #     result = {}
-
-    #     for resource in resources:
-    #         try:
-    #             result[resource] = self.read_resource(resource)
-    #         except Exception as e:
-    #             self.logger.warning(f"Could not read resource {resource}: {str(e)}")
-
-    #     return result
-
-    # def load_from_paths(
-    #     self, source_data_path: List[Union[str, Path]], **kwargs
-    # ) -> Union[
-    #     pd.DataFrame, gpd.GeoDataFrame, Dict[str, Union[pd.DataFrame, gpd.GeoDataFrame]]
-    # ]:
-    #     """Load data from paths"""
-    #     if len(source_data_path) == 1:
-    #         return self.read_resource(str(source_data_path[0]))
-    #     else:
-    #         return self.read_all_resources()
-
 
 class HDXHandler(BaseHandler):
     """Handler for HDX datasets"""

gigaspatial/handlers/maxar_image.py

@@ -14,7 +14,6 @@ from gigaspatial.processing.geo import (
     convert_to_geodataframe,
     buffer_geodataframe,
 )
-from gigaspatial.processing.sat_images import calculate_pixels_at_location
 from gigaspatial.config import config as global_config
 
 
@@ -142,7 +141,7 @@ class MaxarImageDownloader:
                 self.logger.warning(
                     f"Attempt {attempt + 1} of downloading {output_path.name} failed: {str(e)}"
                 )
-                if attempt < self.max_retries - 1:
+                if attempt < self.config.max_retries - 1:
                     sleep(self.config.retry_delay)
                 else:
                     self.logger.warning(