giga-spatial 0.6.4__py3-none-any.whl → 0.6.6__py3-none-any.whl

{giga_spatial-0.6.4.dist-info → giga_spatial-0.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: giga-spatial
-Version: 0.6.4
+Version: 0.6.6
 Summary: A package for spatial data download & processing
 Home-page: https://github.com/unicef/giga-spatial
 Author: Utku Can Ozturk
@@ -31,6 +31,8 @@ Requires-Dist: OWSLib==0.32.1
 Requires-Dist: pydantic-settings>=2.7.1
 Requires-Dist: hdx-python-api>=6.3.8
 Requires-Dist: bs4==0.0.2
+Requires-Dist: sqlalchemy-trino==0.5.0
+Requires-Dist: dask>=2024.12.1
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

giga_spatial-0.6.6.dist-info/RECORD

@@ -0,0 +1,50 @@
+giga_spatial-0.6.6.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+gigaspatial/__init__.py,sha256=I3h5MyD10PkOUQEBnR6L9ja7s4WeTEg8rRjRKTCWYWQ,22
+gigaspatial/config.py,sha256=pLbxGc08OHT2IfTBzZVuIJTPR2vvg3KTFfvciOtRswk,9304
+gigaspatial/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gigaspatial/core/io/__init__.py,sha256=stlpgEeHf5KIb2ZW8yEbdJK5iq6n_wX4DPmKyR9PK-w,317
+gigaspatial/core/io/adls_data_store.py,sha256=Zv-D_8d_2h57HnCUTJb0JWWjXqR_0XH4F8Nu_UFZK9E,11975
+gigaspatial/core/io/data_api.py,sha256=0TtXEfqIz9m1uC8ktcUhZZtx6ZIPY_SIXNnlIodcry8,3970
+gigaspatial/core/io/data_store.py,sha256=mi8fy78Dtwj4dpKkyDM6kTlna1lfCQ5ro2hUAOFr83A,3223
+gigaspatial/core/io/database.py,sha256=kR9ZHuIpZEjXmyj_PnMn6ManQ504kl_f1kJRjHuNWwk,11378
+gigaspatial/core/io/local_data_store.py,sha256=hcu7DNYa3AL6sEPMqguzxWal_bnP7CIpbwpoiyf5TCw,2933
+gigaspatial/core/io/readers.py,sha256=gqFKGRCsAP_EBXipqGtT8MEV-x0u6SrCqaSiOC5YPTA,9284
+gigaspatial/core/io/writers.py,sha256=asb56ZHQEWO2rdilIq7QywDRk8yfebecWv1KwzUpaXI,4367
+gigaspatial/core/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gigaspatial/core/schemas/entity.py,sha256=QAhEW0-JgdWh9pjKGbB5ArvqtVK85ayYZJPgjdb-IKw,8590
+gigaspatial/generators/__init__.py,sha256=cKbMTW7Eh-oDPtM9OfGP14_ekVwc2_7Je7n_pr_anig,223
+gigaspatial/generators/poi.py,sha256=FMYH2yPFuCeYiwj-pHYe83bAxXPkQP28VDQx5L2eEnQ,35869
+gigaspatial/generators/zonal/__init__.py,sha256=egnpvGVeIOS2Zg516AT84tJnIqS4owxmMLLmBQJmK7Y,301
+gigaspatial/generators/zonal/admin.py,sha256=rgOyQX3f_g9qnXqrf-NkR2GEdwOqjNuPNe1H7AUVsfg,3698
+gigaspatial/generators/zonal/base.py,sha256=aoq9AmawPnxu04YI7hPmJzvBTyuUqBQuMZAEUdiVStg,22818
+gigaspatial/generators/zonal/geometry.py,sha256=JbaQ4WS6g45g3uBaATfJd7DS3wW-GPW0GW9FtsvGH-c,21284
+gigaspatial/generators/zonal/mercator.py,sha256=fA02j30PWB5BVjrbNGCMjiOw-ds182yK7R27z8mWFug,5291
+gigaspatial/grid/__init__.py,sha256=ypSSyZ4fYtMNc4IG7chSD7NkUfS2bv9KWRsKR1D9pDI,80
+gigaspatial/grid/mercator_tiles.py,sha256=mAYZDBJ1U0l3z9i4rh5OqiPhOGWcBYzUOI1cvQG_Ff4,11240
+gigaspatial/handlers/__init__.py,sha256=T0_6OxXQ59yTu9g2P6P9vnepudOWp_85R4WQKxRF94c,1618
+gigaspatial/handlers/base.py,sha256=ZcahOEMmS_uECBqOhEEdCoDcPCbVItA5mRS5zEUqR-s,27194
+gigaspatial/handlers/boundaries.py,sha256=jtWyQt3iAzS77mbAOi7mjh3cv_YCV3uB_r1h56gCfeY,20729
+gigaspatial/handlers/ghsl.py,sha256=aSEVQVANzJf8O8TiQYmfwyeM43ZaO65VJHmiuLSQfLs,30524
+gigaspatial/handlers/giga.py,sha256=F5ZfcE37a24X-c6Xhyt72C9eZZbyN_gV7w_InxKFMQQ,28348
+gigaspatial/handlers/google_open_buildings.py,sha256=Liqk7qJhDtB4Ia4uhBe44LFcf-XVKBjRfj-pWlE5erY,16594
+gigaspatial/handlers/hdx.py,sha256=LTEs_xZF1yPhD8dAdZ_YN8Vcan7iB5_tZ8NjF_ip6u0,18001
+gigaspatial/handlers/mapbox_image.py,sha256=M_nkJ_b1PD8FG1ajVgSycCb0NRTAI_SLpHdzszNetKA,7786
+gigaspatial/handlers/maxar_image.py,sha256=kcc8uGljQB0Yh0MKBA7lT7KwBbNZwFzuyBklR3db1P4,10204
+gigaspatial/handlers/microsoft_global_buildings.py,sha256=bQ5WHIv3v0wWrZZUbZkKPRjgdlqIxlK7CV_0zSvdrTw,20292
+gigaspatial/handlers/ookla_speedtest.py,sha256=EcvSAxJZ9GPfzYnT_C85Qgy2ecc9ndf70Pklk53OdC8,6506
+gigaspatial/handlers/opencellid.py,sha256=KuJqd-5-RO5ZzyDaBSrTgCK2ib5N_m3RUcPlX5heWwI,10683
+gigaspatial/handlers/osm.py,sha256=sLNMkOVh1v50jrWw7Z0-HILY5QTQjgKCHCeAfXj5jA8,14084
+gigaspatial/handlers/overture.py,sha256=lKeNw00v5Qia7LdWORuYihnlKEqxE9m38tdeRrvag9k,4218
+gigaspatial/handlers/rwi.py,sha256=eAaplDysVeBhghJusYUKZYbKL5hW-klWvi8pWhILQkY,4962
+gigaspatial/handlers/unicef_georepo.py,sha256=ODYNvkU_UKgOHXT--0MqmJ4Uk6U1_mp9xgehbTzKpX8,31924
+gigaspatial/handlers/worldpop.py,sha256=pkTmqb0k0vpa58t6tM3jfcpMHt1YuayLPFEFEULlrLs,30156
+gigaspatial/processing/__init__.py,sha256=QDVL-QbLCrIb19lrajP7LrHNdGdnsLeGcvAs_jQpdRM,183
+gigaspatial/processing/algorithms.py,sha256=6fBCwbZrI_ISWJ7UpkH6moq1vw-7dBy14yXSLHZprqY,6591
+gigaspatial/processing/geo.py,sha256=8kD7-LQdGzKVfuZDWr3zK5uQhPzgxbZ3JBPosLRBJ5M,41390
+gigaspatial/processing/sat_images.py,sha256=YUbH5MFNzl6NX49Obk14WaFcr1s3SyGJIOk-kRpbBNg,1429
+gigaspatial/processing/tif_processor.py,sha256=QLln9D-_zBhdYQL9NAL_bmo0bmmxE3sxDUQEglYQK94,27490
+gigaspatial/processing/utils.py,sha256=HC85vGKQakxlkoQAkZmeAXWHsenAwTIRn7jPKUA7x20,1500
+giga_spatial-0.6.6.dist-info/METADATA,sha256=ZKoXmthabbL_5xJYHdQfk3ev4Dz02tWU6RAtpv0vWSU,7537
+giga_spatial-0.6.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+giga_spatial-0.6.6.dist-info/top_level.txt,sha256=LZsccgw6H4zXT7m6Y4XChm-Y5LjHAwZ2hkGN_B3ExmI,12
+giga_spatial-0.6.6.dist-info/RECORD,,

gigaspatial/__init__.py

@@ -1 +1 @@
-__version__ = "0.6.4"
+__version__ = "0.6.6"

gigaspatial/config.py

@@ -70,11 +70,12 @@ class Config(BaseSettings):
         description="Directory for temporary/cache files",
         alias="CACHE_DIR",
     )
-    ADMIN_BOUNDARIES_DATA_DIR: Path = Field(
-        default=Path("admin_boundaries"),
+    ADMIN_BOUNDARIES_DATA_DIR: Optional[Path] = Field(
+        default=None,
         description="Root directory for administrative boundary data",
         alias="ADMIN_BOUNDARIES_DIR",
     )
+    DB_CONFIG: Optional[Dict] = Field(default=None, alias="DB_CONFIG")
 
     DATA_TYPES: Dict[str, str] = Field(
         default={
@@ -156,6 +157,11 @@ class Config(BaseSettings):
     ) -> Path:
         """Dynamic path construction for administrative boundary data based on admin level."""
         base_dir = getattr(self, "ADMIN_BOUNDARIES_DATA_DIR")
+        if base_dir is None:
+            raise ValueError(
+                "ADMIN_BOUNDARIES_DATA_DIR is not configured. "
+                "Please set the ADMIN_BOUNDARIES_DIR environment variable."
+            )
         level_dir = f"admin{admin_level}"
         file = f"{country_code}_{level_dir}{file_suffix}"
 
@@ -174,7 +180,6 @@ class Config(BaseSettings):
         "SILVER_DATA_DIR",
         "GOLD_DATA_DIR",
         "CACHE_DIR",
-        "ADMIN_BOUNDARIES_DATA_DIR",
         mode="before",
     )
     def resolve_and_validate_paths(
@@ -192,10 +197,30 @@ class Config(BaseSettings):
         resolved = path.expanduser().resolve()
         return resolved if resolve else path
 
+    @field_validator("ADMIN_BOUNDARIES_DATA_DIR", mode="before")
+    def validate_admin_boundaries_dir(
+        cls, value: Union[str, Path, None]
+    ) -> Optional[Path]:
+        """Validator for ADMIN_BOUNDARIES_DATA_DIR that handles None and string values."""
+        if value is None:
+            return None
+        if isinstance(value, str):
+            return Path(value)
+        elif isinstance(value, Path):
+            return value
+        else:
+            raise ValueError(
+                f"Invalid path type for ADMIN_BOUNDARIES_DATA_DIR: {type(value)}"
+            )
+
     def ensure_directories_exist(self, create: bool = False) -> None:
         """Ensures all configured directories exist."""
         for field_name, field_value in self.__dict__.items():
-            if isinstance(field_value, Path) and not field_value.exists():
+            if (
+                isinstance(field_value, Path)
+                and field_value is not None
+                and not field_value.exists()
+            ):
                 if create:
                     field_value.mkdir(parents=True, exist_ok=True)
                 else:

gigaspatial/core/io/__init__.py

@@ -1,5 +1,6 @@
 from gigaspatial.core.io.adls_data_store import ADLSDataStore
 from gigaspatial.core.io.local_data_store import LocalDataStore
 from gigaspatial.core.io.data_api import GigaDataAPI
+from gigaspatial.core.io.database import DBConnection
 from gigaspatial.core.io.readers import *
 from gigaspatial.core.io.writers import *

gigaspatial/core/io/data_api.py

@@ -35,7 +35,9 @@ class GigaDataAPI:
         :param sort: Whether to sort the country list alphabetically (default is True).
         """
         country_list = [
-            t.name for t in self.client.list_all_tables() if t.share == self.share_name
+            t.name
+            for t in self.client.list_all_tables()
+            if t.schema == self.schema_name
         ]
         if sort:
             country_list.sort()

gigaspatial/core/io/database.py

@@ -0,0 +1,319 @@
+from typing import List, Dict, Optional, Union, Literal
+
+import pandas as pd
+import dask.dataframe as dd
+
+from sqlalchemy import inspect, MetaData, Table, select, create_engine, event, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.exc import SQLAlchemyError
+from urllib.parse import quote_plus
+import warnings
+
+from gigaspatial.config import config as global_config
+
+
+class DBConnection:
+    """
+    A unified database connection class supporting both Trino and PostgreSQL.
+    """
+
+    DB_CONFIG = global_config.DB_CONFIG or {}
+
+    def __init__(
+        self,
+        db_type: Literal["postgresql", "trino"] = DB_CONFIG.get(
+            "db_type", "postgresql"
+        ),
+        host: Optional[str] = DB_CONFIG.get("host", None),
+        port: Union[int, str] = DB_CONFIG.get("port", None),  # type: ignore
+        user: Optional[str] = DB_CONFIG.get("user", None),
+        password: Optional[str] = DB_CONFIG.get("password", None),
+        catalog: Optional[str] = DB_CONFIG.get("catalog", None),  # For Trino
+        database: Optional[str] = DB_CONFIG.get("database", None),  # For PostgreSQL
+        schema: str = DB_CONFIG.get("schema", "public"),  # Default for PostgreSQL
+        http_scheme: str = DB_CONFIG.get("http_scheme", "https"),  # For Trino
+        sslmode: str = DB_CONFIG.get("sslmode", "require"),  # For PostgreSQL
+        **kwargs,
+    ):
+        """
+        Initialize a database connection for either Trino or PostgreSQL.
+
+        Args:
+            db_type: Either "trino" or "postgresql"
+            host: Database server host
+            port: Database server port
+            user: Username
+            password: Password
+            catalog: Trino catalog name
+            database: PostgreSQL database name
+            schema: Default schema name
+            http_scheme: For Trino ("http" or "https")
+            sslmode: For PostgreSQL (e.g., "require", "verify-full")
+            **kwargs: Additional connection parameters
+        """
+        self.db_type = db_type.lower()
+        self.host = host
+        self.port = str(port) if port else None
+        self.user = user
+        self.password = quote_plus(password) if password else None
+        self.default_schema = schema
+
+        if self.db_type == "trino":
+            self.catalog = catalog
+            self.http_scheme = http_scheme
+            self.engine = self._create_trino_engine(**kwargs)
+        elif self.db_type == "postgresql":
+            self.database = database
+            self.sslmode = sslmode
+            self.engine = self._create_postgresql_engine(**kwargs)
+        else:
+            raise ValueError(f"Unsupported database type: {db_type}")
+
+        self._add_event_listener()
+
+    def _create_trino_engine(self, **kwargs) -> Engine:
+        """Create a Trino SQLAlchemy engine."""
+        self._connection_string = (
+            f"trino://{self.user}:{self.password}@{self.host}:{self.port}/"
+            f"{self.catalog}/{self.default_schema}"
+        )
+        return create_engine(
+            self._connection_string,
+            connect_args={"http_scheme": self.http_scheme},
+            **kwargs,
+        )
+
+    def _create_postgresql_engine(self, **kwargs) -> Engine:
+        """Create a PostgreSQL SQLAlchemy engine."""
+        self._connection_string = (
+            f"postgresql://{self.user}:{self.password}@{self.host}:{self.port}/"
+            f"{self.database}?sslmode={self.sslmode}"
+        )
+        return create_engine(self._connection_string, **kwargs)
+
+    def _add_event_listener(self):
+        """Add event listeners for schema setting."""
+        if self.db_type == "trino":
+
+            @event.listens_for(self.engine, "connect", insert=True)
+            def set_current_schema(dbapi_connection, connection_record):
+                cursor_obj = dbapi_connection.cursor()
+                try:
+                    cursor_obj.execute(f"USE {self.default_schema}")
+                except Exception as e:
+                    warnings.warn(f"Could not set schema to {self.default_schema}: {e}")
+                finally:
+                    cursor_obj.close()
+
+    def get_connection_string(self) -> str:
+        """
+        Returns the connection string used to create the engine.
+
+        Returns:
+            str: The connection string.
+        """
+        return self._connection_string
+
+    def get_schema_names(self) -> List[str]:
+        """Get list of all schema names."""
+        inspector = inspect(self.engine)
+        return inspector.get_schema_names()
+
+    def get_table_names(self, schema: Optional[str] = None) -> List[str]:
+        """Get list of table names in a schema."""
+        schema = schema or self.default_schema
+        inspector = inspect(self.engine)
+        return inspector.get_table_names(schema=schema)
+
+    def get_view_names(self, schema: Optional[str] = None) -> List[str]:
+        """Get list of view names in a schema."""
+        schema = schema or self.default_schema
+        inspector = inspect(self.engine)
+        return inspector.get_view_names(schema=schema)
+
+    def get_column_names(
+        self, table_name: str, schema: Optional[str] = None
+    ) -> List[str]:
+        """Get column names for a specific table."""
+        if "." in table_name:
+            schema, table_name = table_name.split(".")
+        else:
+            schema = schema or self.default_schema
+
+        inspector = inspect(self.engine)
+        columns = inspector.get_columns(table_name, schema=schema)
+        return [col["name"] for col in columns]
+
+    def get_table_info(
+        self, table_name: str, schema: Optional[str] = None
+    ) -> List[Dict]:
+        """Get detailed column information for a table."""
+        if "." in table_name:
+            schema, table_name = table_name.split(".")
+        else:
+            schema = schema or self.default_schema
+
+        inspector = inspect(self.engine)
+        return inspector.get_columns(table_name, schema=schema)
+
+    def get_primary_keys(
+        self, table_name: str, schema: Optional[str] = None
+    ) -> List[str]:
+        """Get primary key columns for a table."""
+        if "." in table_name:
+            schema, table_name = table_name.split(".")
+        else:
+            schema = schema or self.default_schema
+
+        inspector = inspect(self.engine)
+        try:
+            return inspector.get_pk_constraint(table_name, schema=schema)[
+                "constrained_columns"
+            ]
+        except:
+            return []  # Some databases may not support PK constraints
+
+    def table_exists(self, table_name: str, schema: Optional[str] = None) -> bool:
+        """Check if a table exists."""
+        if "." in table_name:
+            schema, table_name = table_name.split(".")
+        else:
+            schema = schema or self.default_schema
+
+        return table_name in self.get_table_names(schema=schema)
+
+    # PostgreSQL-specific methods
+    def get_extensions(self) -> List[str]:
+        """Get list of installed PostgreSQL extensions (PostgreSQL only)."""
+        if self.db_type != "postgresql":
+            raise NotImplementedError(
+                "This method is only available for PostgreSQL connections"
+            )
+
+        with self.engine.connect() as conn:
+            result = conn.execute("SELECT extname FROM pg_extension")
+            return [row[0] for row in result]
+
+    def execute_query(
+        self, query: str, fetch_results: bool = True, params: Optional[Dict] = None
+    ) -> Union[List[tuple], None]:
+        """
+        Executes a SQL query (works for both PostgreSQL and Trino).
+
+        Args:
+            query: SQL query to execute
+            fetch_results: Whether to fetch results
+            params: Parameters for parameterized queries
+
+        Returns:
+            Results as list of tuples or None
+        """
+        try:
+            with self.engine.connect() as connection:
+                stmt = text(query)
+                result = (
+                    connection.execute(stmt, params)
+                    if params
+                    else connection.execute(stmt)
+                )
+
+                if fetch_results and result.returns_rows:
+                    return result.fetchall()
+                return None
+        except SQLAlchemyError as e:
+            print(f"Error executing query: {e}")
+            raise
+
+    def test_connection(self) -> bool:
+        """
+        Tests the database connection (works for both PostgreSQL and Trino).
+
+        Returns:
+            True if connection successful, False otherwise
+        """
+        test_query = (
+            "SELECT 1"
+            if self.db_type == "postgresql"
+            else "SELECT 1 AS connection_test"
+        )
+
+        try:
+            print(
+                f"Attempting to connect to {self.db_type} at {self.host}:{self.port}..."
+            )
+            with self.engine.connect() as conn:
+                conn.execute(text(test_query))
+            print(f"Successfully connected to {self.db_type.upper()}.")
+            return True
+        except Exception as e:
+            print(f"Failed to connect to {self.db_type.upper()}: {e}")
+            return False
+
+    def read_sql_to_dataframe(
+        self, query: str, params: Optional[Dict] = None
+    ) -> pd.DataFrame:
+        """
+        Executes query and returns results as pandas DataFrame (works for both).
+
+        Args:
+            query: SQL query to execute
+            params: Parameters for parameterized queries
+
+        Returns:
+            pandas DataFrame with results
+        """
+        try:
+            with self.engine.connect() as connection:
+                return pd.read_sql_query(text(query), connection, params=params)
+        except SQLAlchemyError as e:
+            print(f"Error reading SQL to DataFrame: {e}")
+            raise
+
+    def read_sql_to_dask_dataframe(
+        self,
+        table_name: str,
+        index_col: str,
+        columns: Optional[List[str]] = None,
+        limit: Optional[int] = None,
+        **kwargs,
+    ) -> pd.DataFrame:
+        """
+        Reads data to Dask DataFrame (works for both, but connection string differs).
+
+        Args:
+            table_name: Table name (schema.table or just table)
+            columns: List of columns to select
+            limit: Maximum rows to return
+            **kwargs: Additional arguments
+
+        Returns:
+            Dask DataFrame with results
+        """
+        try:
+            connection_string = self.get_connection_string()
+
+            # Handle schema.table format
+            if "." in table_name:
+                schema, table = table_name.split(".")
+            else:
+                schema = self.default_schema
+                table = table_name
+
+            metadata = MetaData()
+            table_obj = Table(table, metadata, schema=schema, autoload_with=self.engine)
+
+            # Build query
+            query = (
+                select(*[table_obj.c[col] for col in columns])
+                if columns
+                else select(table_obj)
+            )
+            if limit:
+                query = query.limit(limit)
+
+            return dd.read_sql_query(
+                sql=query, con=connection_string, index_col=index_col, **kwargs
+            )
+        except Exception as e:
+            print(f"Error reading SQL to Dask DataFrame: {e}")
+            raise ValueError(f"Failed to read SQL to Dask DataFrame: {e}") from e

gigaspatial/generators/__init__.py

@@ -1,2 +1,6 @@
 from gigaspatial.generators.poi import PoiViewGenerator, PoiViewGeneratorConfig
-from gigaspatial.generators.zonal import GeometryBasedZonalViewGenerator
+from gigaspatial.generators.zonal import (
+    GeometryBasedZonalViewGenerator,
+    MercatorViewGenerator,
+    AdminBoundariesViewGenerator,
+)