giga-spatial 0.6.3__tar.gz → 0.6.5__tar.gz

{giga_spatial-0.6.3 → giga_spatial-0.6.5}/.env_sample

@@ -4,6 +4,7 @@ export GOOGLE_SERVICE_ACCOUNT=""
 export API_PROFILE_FILE_PATH=""
 export API_SHARE_NAME=""
 export API_SCHEMA_NAME=""
+export DB_CONFIG='{}'
 export MAPBOX_ACCESS_TOKEN=""
 export MAXAR_USERNAME=""
 export MAXAR_PASSWORD=""
@@ -12,6 +13,8 @@ export OPENCELLID_ACCESS_TOKEN=""
 export GEOREPO_API_KEY=""
 export GEOREPO_USER_EMAIL=""
 export GIGA_SCHOOL_LOCATION_API_KEY=""
+export GIGA_SCHOOL_PROFILE_API_KEY=""
+export GIGA_SCHOOL_MEASUREMENTS_API_KEY=""
 export ROOT_DATA_DIR=""
 export BRONZE_DIR=""
 export SILVER_DIR=""

{giga_spatial-0.6.3 → giga_spatial-0.6.5}/CHANGELOG.md

@@ -2,6 +2,124 @@
 
 All notable changes to this project will be documented in this file.
 
+## [v0.6.5] - 2025-07-01
+
+### Added
+
+- **`MercatorTiles.get_quadkeys_from_points()`**  
+  New static method for efficient 1:1 point-to-quadkey mapping using coordinate-based logic, improving performance over spatial joins.
+
+- **`AdminBoundariesViewGenerator`**  
+  New generator class for producing zonal views based on administrative boundaries (e.g., districts, provinces) with flexible source and admin level support.
+
+- **Zonal View Generator Enhancements**  
+  - `_view`: Internal attribute for accumulating mapped statistics.  
+  - `view`: Exposes current state of zonal view.  
+  - `add_variable_to_view()`: Adds mapped data from `map_points`, `map_polygons`, or `map_rasters` with robust validation and zone alignment.  
+  - `to_dataframe()` and `to_geodataframe()` methods added for exporting current view in tabular or spatial formats.
+
+- **`PoiViewGenerator` Enhancements**  
+  - Consistent `_view` DataFrame for storing mapped results.  
+  - `_update_view()`: Central method to update POI data.  
+  - `save_view()`: Improved format handling (CSV, Parquet, GeoJSON, etc.) with geometry recovery.  
+  - `to_dataframe()` and `to_geodataframe()` methods added for convenient export of enriched POI view.  
+  - Robust duplicate ID detection and CRS validation in `map_zonal_stats`.
+
+- **`TifProcessor` Enhancements**  
+  - `sample_by_polygons_batched()`: Parallel polygon sampling.  
+  - Enhanced `sample_by_polygons()` with nodata masking and multiple stats.  
+  - `warn_on_error`: Flag to suppress sampling warnings.
+
+- **GeoTIFF Multi-Band Support**  
+  - `multi` mode added for multi-band raster support.  
+  - Auto-detects band names via metadata.  
+  - Strict validation of band count based on mode (`single`, `rgb`, `rgba`, `multi`).
+
+- **Spatial Distance Graph Algorithm**  
+  - `build_distance_graph()` added for fast KD-tree-based spatial matching.  
+  - Supports both `DataFrame` and `GeoDataFrame` inputs.  
+  - Outputs a `networkx.Graph` with optional DataFrame of matches.  
+  - Handles projections, self-match exclusion, and includes verbose stats/logs.
+
+- **Database Integration (Experimental)**  
+  - Added `DBConnection` class in `core/io/database.py` for unified Trino and PostgreSQL access.  
+  - Supports schema/table introspection, query execution, and reading into `pandas` or `dask`.  
+  - Handles connection creation, credential management, and diagnostics.  
+  - Utility methods for schema/view/table/column listings and parameterized queries.
+
+- **GHSL Population Mapping**  
+  - `map_ghsl_pop()` method added to `GeometryBasedZonalViewGenerator`.  
+  - Aggregates GHSL population rasters to user-defined zones.  
+  - Supports `intersects` and `fractional` predicates (latter for 1000m resolution only).  
+  - Returns population statistics (e.g., `sum`) with customizable column prefix.
+
+### Changed
+
+- **`MercatorTiles.from_points()`** now internally uses `get_quadkeys_from_points()` for better performance.
+
+- **`map_points()` and `map_rasters()`** now return `Dict[zone_id, value]` to support direct usage with `add_variable_to_view()`.
+
+- **Refactored `aggregate_polygons_to_zones()`**  
+  - `area_weighted` deprecated in favor of `predicate`.  
+  - Supports flexible predicates like `"within"`, `"fractional"` for spatial aggregation.  
+  - `map_polygons()` updated to reflect this change.
+
+- **Optional Admin Boundaries Configuration**  
+  - `ADMIN_BOUNDARIES_DATA_DIR` is now optional.  
+  - `AdminBoundaries.create()` only attempts to load if explicitly configured or path is provided.  
+  - Improved documentation and fallback behavior for missing configs.
+
+### Fixed
+
+- **GHSL Downloader**  
+  - ZIP files are now downloaded into a temporary cache directory using `requests.get()`.  
+  - Avoids unnecessary writes and ensures cleanup.
+
+- **`TifProcessor`**  
+  - Removed polygon sampling warnings unless explicitly enabled.
+
+### Deprecated
+
+- `TifProcessor.tabular` → use `to_dataframe()` instead.  
+- `TifProcessor.get_zoned_geodataframe()` → use `to_geodataframe()` instead.  
+- `area_weighted` → use `predicate` in aggregation methods instead.
+
+## [v0.6.4] - 2025-06-19
+
+### Added
+
+- **GigaSchoolProfileFetcher**  
+  - New class to fetch and process school profile data from the Giga School Profile API  
+  - Supports paginated fetching, filtering by country and school ID  
+  - Includes methods to generate connectivity summary statistics by region, connection type, and source
+
+- **GigaSchoolMeasurementsFetcher**  
+  - New class to fetch and process daily real-time connectivity measurements from the Giga API  
+  - Supports filtering by date range and school  
+  - Includes performance summary generation (download/upload speeds, latency, quality flags)
+
+- **AdminBoundaries.from_geoboundaries**  
+  - New class method to download and process geoBoundaries data by country and admin level  
+  - Automatically handles HDX dataset discovery, downloading, and fallback logic
+
+- **HDXConfig.search_datasets**  
+  - Static method to search HDX datasets without full handler initialization  
+  - Supports query string, sort order, result count, HDX site selection, and custom user agent
+
+### Fixed
+
+- Typo in `MaxarImageDownloader` causing runtime error
+
+### Documentation
+
+- **Improved Configuration Guide** (`docs/user-guide/configuration.md`)  
+  - Added comprehensive table of environment variables with defaults and descriptions  
+  - Synced `.env_sample` and `config.py` with docs  
+  - Example `.env` file and guidance on path overrides using `config.set_path`  
+  - New section on `config.ensure_directories_exist` and troubleshooting tips  
+  - Clearer handling of credentials and security notes  
+  - Improved formatting and structure for clarity
+
 ## [v0.6.3] - 2025-06-16
 
 ### Added

{giga_spatial-0.6.3 → giga_spatial-0.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: giga-spatial
-Version: 0.6.3
+Version: 0.6.5
 Summary: A package for spatial data download & processing
 Home-page: https://github.com/unicef/giga-spatial
 Author: Utku Can Ozturk
@@ -31,6 +31,7 @@ Requires-Dist: OWSLib==0.32.1
 Requires-Dist: pydantic-settings>=2.7.1
 Requires-Dist: hdx-python-api>=6.3.8
 Requires-Dist: bs4==0.0.2
+Requires-Dist: sqlalchemy-trino==0.5.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

giga_spatial-0.6.5/docs/changelog.md

@@ -0,0 +1 @@
+--8<-- "CHANGELOG.md"

{giga_spatial-0.6.3 → giga_spatial-0.6.5}/docs/getting-started/quickstart.md

@@ -16,43 +16,36 @@ import gigaspatial as gs
 
 ## Setting Up Configuration
 
-The `gigaspatial` package uses a configuration file (`config.py`) to manage paths, API keys, and other settings. You can customize the configuration as needed.
+The `gigaspatial` package uses a unified configuration system to manage paths, API keys, and other settings.
 
-### Using Environment Variables
+- **Environment Variables:** Most configuration is handled via environment variables, which can be set in a `.env` file at the project root. For a full list of supported variables and their descriptions, see the [Configuration Guide](../user-guide/configuration.md).
+- **Defaults:** If not set, sensible defaults are used for all paths and keys.
+- **Manual Overrides:** You can override data directory paths in your code using `config.set_path`.
 
-The package can read configuration settings from an environment file (e.g., `.env`). Here's an example of how to set up the `.env` file based on the `env_sample`:
+### Example `.env` File
 
 ```bash
-# Paths for different data types
 BRONZE_DIR=/path/to/your/bronze_tier_data
 SILVER_DIR=/path/to/your/silver_tier_data
 GOLD_DIR=/path/to/your/gold_tier_data
 VIEWS_DIR=/path/to/your/views_data
-ADMIN_BOUNDARIES_DIR=/path/to/your/admin_boundaries_data
-
-# API keys and tokens
+CACHE_DIR=/path/to/your/cache
+ADMIN_BOUNDARIES_DIR=/path/to/your/admin_boundaries
 MAPBOX_ACCESS_TOKEN=your_mapbox_token_here
-MAXAR_USERNAME=your_maxar_username_here
-MAXAR_PASSWORD=your_maxar_password_here
-MAXAR_CONNECTION_STRING=your_maxar_key_here
+# ... other keys ...
 ```
 
-The `config.py` file will automatically read these environment variables and set the paths and keys accordingly.
-
-### Setting Paths Manually
-
-You can also set paths manually in your code:
+### Setting Paths Programmatically
 
 ```python
 from gigaspatial.config import config
 
-# Example: Setting custom data storage paths
 config.set_path("bronze", "/path/to/your/bronze_tier_data")
 config.set_path("gold", "/path/to/your/gold_tier_data")
 config.set_path("views", "/path/to/your/views_data")
 ```
 
-API keys and tokens should be set through environment variables.
+> For more details and troubleshooting, see the [full configuration guide](../user-guide/configuration.md).
 
 ## Downloading and Processing Geospatial Data
 

giga_spatial-0.6.5/docs/user-guide/configuration.md

@@ -0,0 +1,130 @@
+# Configuration
+
+The `gigaspatial` package uses a unified configuration system, managed by the `config.py` file, to handle paths, API keys, and other settings. This guide explains how to configure the package for your environment.
+
+---
+
+## Environment Variables Overview
+
+Configuration is primarily managed via environment variables, which can be set in a `.env` file at the project root. Below is a table of all supported environment variables, their defaults, and descriptions:
+
+| Variable                        | Default         | Description                                      |
+|----------------------------------|-----------------|--------------------------------------------------|
+| ADLS_CONNECTION_STRING           | ""              | Azure Data Lake connection string                 |
+| ADLS_CONTAINER_NAME              | ""              | Azure Data Lake container name                    |
+| GOOGLE_SERVICE_ACCOUNT           | ""              | Google service account credentials                |
+| API_PROFILE_FILE_PATH            | profile.share   | Path to API profile file                          |
+| API_SHARE_NAME                   | ""              | API share name                                    |
+| API_SCHEMA_NAME                  | ""              | API schema name                                   |
+| MAPBOX_ACCESS_TOKEN              | ""              | Mapbox API access token                           |
+| MAXAR_USERNAME                   | ""              | Maxar API username                                |
+| MAXAR_PASSWORD                   | ""              | Maxar API password                                |
+| MAXAR_CONNECTION_STRING          | ""              | Maxar API connection string/key                   |
+| OPENCELLID_ACCESS_TOKEN          | ""              | OpenCellID API access token                       |
+| GEOREPO_API_KEY                  | ""              | UNICEF GeoRepo API key                            |
+| GEOREPO_USER_EMAIL               | ""              | UNICEF GeoRepo user email                         |
+| GIGA_SCHOOL_LOCATION_API_KEY     | ""              | GIGA School Location API key                      |
+| GIGA_SCHOOL_PROFILE_API_KEY      | ""              | GIGA School Profile API key                       |
+| GIGA_SCHOOL_MEASUREMENTS_API_KEY | ""              | GIGA School Measurements API key                  |
+| ROOT_DATA_DIR                    | .               | Root directory for all data tiers                 |
+| BRONZE_DIR                       | bronze          | Directory for raw/bronze tier data                |
+| SILVER_DIR                       | silver          | Directory for processed/silver tier data          |
+| GOLD_DIR                         | gold            | Directory for final/gold tier data                |
+| VIEWS_DIR                        | views           | Directory for views data                          |
+| CACHE_DIR                        | cache           | Directory for cache/temp files                    |
+| ADMIN_BOUNDARIES_DIR             | admin_boundaries| Directory for admin boundary data                 |
+
+> **Tip:** You can copy `.env_sample` to `.env` and fill in your values.
+
+---
+
+## Example `.env` File
+
+```bash
+# Data directories
+BRONZE_DIR=/path/to/your/bronze_tier_data
+SILVER_DIR=/path/to/your/silver_tier_data
+GOLD_DIR=/path/to/your/gold_tier_data
+VIEWS_DIR=/path/to/your/views_data
+CACHE_DIR=/path/to/your/cache
+ADMIN_BOUNDARIES_DIR=/path/to/your/admin_boundaries
+
+# API keys and credentials
+MAPBOX_ACCESS_TOKEN=your_mapbox_token_here
+MAXAR_USERNAME=your_maxar_username_here
+MAXAR_PASSWORD=your_maxar_password_here
+MAXAR_CONNECTION_STRING=your_maxar_key_here
+# ... other keys ...
+```
+
+---
+
+## How Configuration is Loaded
+
+- The `config.py` file uses [pydantic-settings](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) to load environment variables from `.env` (if present) or the system environment.
+- All directory paths are resolved as `Path` objects. If a path is relative, it is resolved relative to the current working directory.
+- Defaults are used if environment variables are not set.
+
+---
+
+## Setting Paths and Keys Programmatically
+
+You can override directory paths in your code using the `set_path` method:
+
+```python
+from gigaspatial.config import config
+
+# Set custom data storage paths
+config.set_path("bronze", "/path/to/your/bronze_tier_data")
+config.set_path("gold", "/path/to/your/gold_tier_data")
+config.set_path("views", "/path/to/your/views_data")
+```
+
+> **Note:** API keys and credentials should be set via environment variables for security.
+
+---
+
+## Ensuring Directories Exist
+
+To ensure all configured directories exist (and optionally create them if missing):
+
+```python
+from gigaspatial.config import config
+
+# Raise error if any directory does not exist
+config.ensure_directories_exist(create=False)
+
+# Or, create missing directories automatically
+config.ensure_directories_exist(create=True)
+```
+
+---
+
+## Verifying the Configuration
+
+You can print the current configuration for debugging:
+
+```python
+from gigaspatial.config import config
+print(config)
+```
+
+---
+
+## Troubleshooting
+
+- **.env File Location:** Ensure `.env` is in your project root.
+- **Absolute Paths:** Use absolute paths for directories to avoid confusion.
+- **Environment Variable Precedence:** Values in `.env` override defaults, but can be overridden by system environment variables.
+- **Missing Directories:** Use `config.ensure_directories_exist(create=True)` to create missing directories.
+- **API Keys:** Double-check that all required API keys are set for the services you use.
+
+---
+
+## Next Steps
+
+Once configuration is set up, proceed to the [Data Handling Guide](data-handling/downloading.md) *(Coming Soon)* to start using `gigaspatial`.
+
+---
+
+[Back to User Guide](../index.md)

{giga_spatial-0.6.3 → giga_spatial-0.6.5}/giga_spatial.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: giga-spatial
-Version: 0.6.3
+Version: 0.6.5
 Summary: A package for spatial data download & processing
 Home-page: https://github.com/unicef/giga-spatial
 Author: Utku Can Ozturk
@@ -31,6 +31,7 @@ Requires-Dist: OWSLib==0.32.1
 Requires-Dist: pydantic-settings>=2.7.1
 Requires-Dist: hdx-python-api>=6.3.8
 Requires-Dist: bs4==0.0.2
+Requires-Dist: sqlalchemy-trino==0.5.0
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{giga_spatial-0.6.3 → giga_spatial-0.6.5}/giga_spatial.egg-info/SOURCES.txt

@@ -47,6 +47,7 @@ gigaspatial/core/io/__init__.py
 gigaspatial/core/io/adls_data_store.py
 gigaspatial/core/io/data_api.py
 gigaspatial/core/io/data_store.py
+gigaspatial/core/io/database.py
 gigaspatial/core/io/local_data_store.py
 gigaspatial/core/io/readers.py
 gigaspatial/core/io/writers.py
@@ -55,6 +56,7 @@ gigaspatial/core/schemas/entity.py
 gigaspatial/generators/__init__.py
 gigaspatial/generators/poi.py
 gigaspatial/generators/zonal/__init__.py
+gigaspatial/generators/zonal/admin.py
 gigaspatial/generators/zonal/base.py
 gigaspatial/generators/zonal/geometry.py
 gigaspatial/generators/zonal/mercator.py
@@ -78,6 +80,7 @@ gigaspatial/handlers/rwi.py
 gigaspatial/handlers/unicef_georepo.py
 gigaspatial/handlers/worldpop.py
 gigaspatial/processing/__init__.py
+gigaspatial/processing/algorithms.py
 gigaspatial/processing/geo.py
 gigaspatial/processing/sat_images.py
 gigaspatial/processing/tif_processor.py

giga_spatial-0.6.3/requirements.txt → giga_spatial-0.6.5/giga_spatial.egg-info/requires.txt

@@ -15,4 +15,5 @@ tqdm==4.65.0
 OWSLib==0.32.1
 pydantic-settings>=2.7.1
 hdx-python-api>=6.3.8
-bs4==0.0.2
+bs4==0.0.2
+sqlalchemy-trino==0.5.0

giga_spatial-0.6.5/gigaspatial/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.5"

{giga_spatial-0.6.3 → giga_spatial-0.6.5}/gigaspatial/config.py

@@ -32,6 +32,12 @@ class Config(BaseSettings):
     GIGA_SCHOOL_LOCATION_API_KEY: str = Field(
         default="", alias="GIGA_SCHOOL_LOCATION_API_KEY"
     )
+    GIGA_SCHOOL_PROFILE_API_KEY: str = Field(
+        default="", alias="GIGA_SCHOOL_PROFILE_API_KEY"
+    )
+    GIGA_SCHOOL_MEASUREMENTS_API_KEY: str = Field(
+        default="", alias="GIGA_SCHOOL_MEASUREMENTS_API_KEY"
+    )
 
     ROOT_DATA_DIR: Path = Field(
         default=Path("."),
@@ -64,11 +70,12 @@ class Config(BaseSettings):
         description="Directory for temporary/cache files",
         alias="CACHE_DIR",
     )
-    ADMIN_BOUNDARIES_DATA_DIR: Path = Field(
-        default=Path("admin_boundaries"),
+    ADMIN_BOUNDARIES_DATA_DIR: Optional[Path] = Field(
+        default=None,
         description="Root directory for administrative boundary data",
         alias="ADMIN_BOUNDARIES_DIR",
     )
+    DB_CONFIG: Optional[Dict] = Field(default=None, alias="DB_CONFIG")
 
     DATA_TYPES: Dict[str, str] = Field(
         default={
@@ -150,6 +157,11 @@ class Config(BaseSettings):
     ) -> Path:
         """Dynamic path construction for administrative boundary data based on admin level."""
         base_dir = getattr(self, "ADMIN_BOUNDARIES_DATA_DIR")
+        if base_dir is None:
+            raise ValueError(
+                "ADMIN_BOUNDARIES_DATA_DIR is not configured. "
+                "Please set the ADMIN_BOUNDARIES_DIR environment variable."
+            )
         level_dir = f"admin{admin_level}"
         file = f"{country_code}_{level_dir}{file_suffix}"
 
@@ -168,7 +180,6 @@ class Config(BaseSettings):
         "SILVER_DATA_DIR",
         "GOLD_DATA_DIR",
         "CACHE_DIR",
-        "ADMIN_BOUNDARIES_DATA_DIR",
         mode="before",
     )
     def resolve_and_validate_paths(
@@ -186,10 +197,30 @@ class Config(BaseSettings):
         resolved = path.expanduser().resolve()
         return resolved if resolve else path
 
+    @field_validator("ADMIN_BOUNDARIES_DATA_DIR", mode="before")
+    def validate_admin_boundaries_dir(
+        cls, value: Union[str, Path, None]
+    ) -> Optional[Path]:
+        """Validator for ADMIN_BOUNDARIES_DATA_DIR that handles None and string values."""
+        if value is None:
+            return None
+        if isinstance(value, str):
+            return Path(value)
+        elif isinstance(value, Path):
+            return value
+        else:
+            raise ValueError(
+                f"Invalid path type for ADMIN_BOUNDARIES_DATA_DIR: {type(value)}"
+            )
+
     def ensure_directories_exist(self, create: bool = False) -> None:
         """Ensures all configured directories exist."""
         for field_name, field_value in self.__dict__.items():
-            if isinstance(field_value, Path) and not field_value.exists():
+            if (
+                isinstance(field_value, Path)
+                and field_value is not None
+                and not field_value.exists()
+            ):
                 if create:
                     field_value.mkdir(parents=True, exist_ok=True)
                 else:

{giga_spatial-0.6.3 → giga_spatial-0.6.5}/gigaspatial/core/io/__init__.py

@@ -1,5 +1,6 @@
 from gigaspatial.core.io.adls_data_store import ADLSDataStore
 from gigaspatial.core.io.local_data_store import LocalDataStore
 from gigaspatial.core.io.data_api import GigaDataAPI
+from gigaspatial.core.io.database import DBConnection
 from gigaspatial.core.io.readers import *
 from gigaspatial.core.io.writers import *