giga-spatial 0.6.0__py3-none-any.whl

gigaspatial/config.py

@@ -0,0 +1,226 @@
+from pathlib import Path
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from typing import Optional, Union, Literal, Dict, Any
+import io
+from functools import lru_cache
+import logging
+
+
+class Config(BaseSettings):
+    """
+    Unified configuration with environment variable loading.
+    Manages file system paths for different data tiers (bronze, silver, gold)
+    and their subpaths. All paths can be overridden through environment variables.
+    """
+
+    ADLS_CONNECTION_STRING: str = Field(default="", alias="ADLS_CONNECTION_STRING")
+    ADLS_CONTAINER_NAME: str = Field(default="", alias="ADLS_CONTAINER_NAME")
+    GOOGLE_SERVICE_ACCOUNT: str = Field(default="", alias="GOOGLE_SERVICE_ACCOUNT")
+    API_PROFILE_FILE_PATH: Path = Field(
+        default=Path("profile.share"), alias="API_PROFILE_FILE_PATH"
+    )
+    API_SHARE_NAME: str = Field(default="", alias="API_SHARE_NAME")
+    API_SCHEMA_NAME: str = Field(default="", alias="API_SCHEMA_NAME")
+    MAPBOX_ACCESS_TOKEN: str = Field(default="", alias="MAPBOX_ACCESS_TOKEN")
+    MAXAR_USERNAME: str = Field(default="", alias="MAXAR_USERNAME")
+    MAXAR_PASSWORD: str = Field(default="", alias="MAXAR_PASSWORD")
+    MAXAR_CONNECTION_STRING: str = Field(default="", alias="MAXAR_CONNECTION_STRING")
+    OPENCELLID_ACCESS_TOKEN: str = Field(default="", alias="OPENCELLID_ACCESS_TOKEN")
+    GEOREPO_API_KEY: str = Field(default="", alias="GEOREPO_API_KEY")
+    GEOREPO_USER_EMAIL: str = Field(default="", alias="GEOREPO_USER_EMAIL")
+    GIGA_SCHOOL_LOCATION_API_KEY: str = Field(
+        default="", alias="GIGA_SCHOOL_LOCATION_API_KEY"
+    )
+
+    BRONZE_DATA_DIR: Path = Field(
+        default=Path("bronze"),
+        description="Root directory for raw/bronze tier data",
+        alias="BRONZE_DIR",
+    )
+    SILVER_DATA_DIR: Path = Field(
+        default=Path("silver"),
+        description="Root directory for processed/silver tier data",
+        alias="SILVER_DIR",
+    )
+    GOLD_DATA_DIR: Path = Field(
+        default=Path("gold"),
+        description="Root directory for final/gold tier data",
+        alias="GOLD_DIR",
+    )
+    VIEWS_DATA_DIR: Path = Field(
+        default=Path("views"),
+        description="Root directory for views data",
+        alias="VIEWS_DIR",
+    )
+    CACHE_DIR: Path = Field(
+        default=Path("cache"),
+        description="Directory for temporary/cache files",
+        alias="CACHE_DIR",
+    )
+    ADMIN_BOUNDARIES_DATA_DIR: Path = Field(
+        default=Path("admin_boundaries"),
+        description="Root directory for administrative boundary data",
+        alias="ADMIN_BOUNDARIES_DIR",
+    )
+
+    DATA_TYPES: Dict[str, str] = Field(
+        default={
+            "google_open_buildings": "google_open_buildings",
+            "mapbox_image": "mapbox_images",
+            "microsoft_global_buildings": "microsoft_global_buildings",
+            "ookla_speedtest": "ookla",
+            "srtm": "srtm",
+            "worldpop": "worldpop",
+            "ghsl": "ghsl",
+            "opencellid": "opencellid",
+            "hdx": "hdx",
+            "poi": "poi",
+            "zonal": "zonal",
+        },
+        description="Mapping of data types to directory names",
+    )
+
+    def get_logger(self, name="GigaSpatial", console_level=logging.INFO):
+        logger = logging.getLogger(name)
+        logger.setLevel(logging.INFO)
+
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(console_level)
+
+        LOG_FORMAT = "%(levelname) -10s  %(name) -10s %(asctime) " "-30s: %(message)s"
+
+        formatter = logging.Formatter(LOG_FORMAT)
+        console_handler.setFormatter(formatter)
+
+        if not logger.hasHandlers():
+            logger.addHandler(console_handler)
+
+        return logger
+
+    def get_tqdm_logger_stream(self, logger: logging.Logger, level=logging.INFO):
+        return TqdmToLogger(logger, level=level)
+
+    def set_path(
+        self,
+        tier: Literal["bronze", "silver", "gold", "views"],
+        path: Union[str, Path],
+    ) -> None:
+        """Dynamically set the base path for a given tier."""
+        if tier not in ["bronze", "silver", "gold", "views"]:
+            raise ValueError(
+                f"Invalid tier: {tier}. Must be one of 'bronze', 'silver', 'gold', or 'views'."
+            )
+
+        if isinstance(path, str):
+            path = Path(path)
+
+        setattr(self, f"{tier.upper()}_DATA_DIR", path)
+
+    def get_path(
+        self,
+        data_type: str,
+        tier: Literal["bronze", "silver", "gold", "views"],
+        version: Optional[str] = None,
+    ) -> Path:
+        """Dynamic path construction based on data type and tier."""
+        if tier not in ["bronze", "silver", "gold", "views"]:
+            raise ValueError(
+                f"Invalid tier: {tier}. Must be one of 'bronze', 'silver', 'gold', or 'views'."
+            )
+
+        base_dir = getattr(self, f"{tier.upper()}_DATA_DIR")
+        type_dir = self.DATA_TYPES[data_type]
+        if version:
+            return base_dir / type_dir / version
+        else:
+            return base_dir / type_dir
+
+    def get_admin_path(
+        self,
+        country_code,
+        admin_level: Literal[0, 1, 2, 3, 4],
+        file_suffix: str = ".geojson",
+    ) -> Path:
+        """Dynamic path construction for administrative boundary data based on admin level."""
+        base_dir = getattr(self, "ADMIN_BOUNDARIES_DATA_DIR")
+        level_dir = f"admin{admin_level}"
+        file = f"{country_code}_{level_dir}{file_suffix}"
+
+        return base_dir / level_dir / file
+
+    model_config = SettingsConfigDict(
+        env_file=".env",
+        env_prefix="",
+        validate_assignment=True,
+        case_sensitive=True,
+        extra="allow",
+    )
+
+    @field_validator(
+        "BRONZE_DATA_DIR",
+        "SILVER_DATA_DIR",
+        "GOLD_DATA_DIR",
+        "CACHE_DIR",
+        "ADMIN_BOUNDARIES_DATA_DIR",
+        mode="before",
+    )
+    def resolve_and_validate_paths(
+        cls, value: Union[str, Path], resolve=False
+    ) -> Union[Path, Any]:
+        """Smart validator that only processes Path fields"""
+
+        if isinstance(value, str):
+            path = Path(value)
+        elif isinstance(value, Path):
+            path = value
+        else:
+            raise ValueError(f"Invalid path type for {field.name}: {type(value)}")
+
+        resolved = path.expanduser().resolve()
+        return resolved if resolve else path
+
+    def ensure_directories_exist(self, create: bool = False) -> None:
+        """Ensures all configured directories exist."""
+        for field_name, field_value in self.__dict__.items():
+            if isinstance(field_value, Path) and not field_value.exists():
+                if create:
+                    field_value.mkdir(parents=True, exist_ok=True)
+                else:
+                    raise FileNotFoundError(f"Directory does not exist: {field_value}")
+
+
+class TqdmToLogger(io.StringIO):
+    """
+    File-like object to redirect tqdm output to a logger.
+    """
+
+    def __init__(self, logger, level=logging.INFO):
+        super().__init__()
+        self.logger = logger
+        self.level = level
+        self.buf = ""  # To store partial writes
+
+    def write(self, buf):
+        # tqdm often writes partial lines, and then a full line with \r
+        # We accumulate buffer and only log when a full line (or significant update) is received
+        self.buf += buf
+        if "\r" in buf or "\n" in buf:  # Heuristic for a "full" update
+            self.logger.log(self.level, self.buf.strip("\r\n"))
+            self.buf = ""  # Reset buffer after logging
+
+    def flush(self):
+        # Ensure any remaining buffer is logged on flush
+        if self.buf:
+            self.logger.log(self.level, self.buf.strip("\r\n"))
+            self.buf = ""
+
+
+@lru_cache()
+def get_default_config() -> Config:
+    """Returns a singleton instance of Config."""
+    return Config()
+
+
+# Singleton instance
+config = get_default_config()

gigaspatial/core/io/__init__.py

@@ -0,0 +1,5 @@
+from gigaspatial.core.io.adls_data_store import ADLSDataStore
+from gigaspatial.core.io.local_data_store import LocalDataStore
+from gigaspatial.core.io.data_api import GigaDataAPI
+from gigaspatial.core.io.readers import *
+from gigaspatial.core.io.writers import *

gigaspatial/core/io/adls_data_store.py

@@ -0,0 +1,325 @@
+from azure.storage.blob import BlobServiceClient
+import io
+import contextlib
+import logging
+import os
+from typing import Union, Optional
+
+from .data_store import DataStore
+from gigaspatial.config import config
+
+logging.getLogger("azure.core.pipeline.policies.http_logging_policy").setLevel(
+    logging.WARNING
+)
+
+
+class ADLSDataStore(DataStore):
+    """
+    An implementation of DataStore for Azure Data Lake Storage.
+    """
+
+    def __init__(
+        self,
+        container: str = config.ADLS_CONTAINER_NAME,
+        connection_string: str = config.ADLS_CONNECTION_STRING,
+    ):
+        """
+        Create a new instance of ADLSDataStore
+        :param container: The name of the container in ADLS to interact with.
+        """
+        self.blob_service_client = BlobServiceClient.from_connection_string(
+            connection_string
+        )
+        self.container_client = self.blob_service_client.get_container_client(
+            container=container
+        )
+        self.container = container
+
+    def read_file(self, path: str, encoding: Optional[str] = None) -> Union[str, bytes]:
+        """
+        Read file with flexible encoding support.
+
+        :param path: Path to the file in blob storage
+        :param encoding: File encoding (optional)
+        :return: File contents as string or bytes
+        """
+        try:
+            blob_client = self.container_client.get_blob_client(path)
+            blob_data = blob_client.download_blob().readall()
+
+            # If no encoding specified, return raw bytes
+            if encoding is None:
+                return blob_data
+
+            # If encoding is specified, decode the bytes
+            return blob_data.decode(encoding)
+
+        except Exception as e:
+            raise IOError(f"Error reading file {path}: {e}")
+
+    def write_file(self, path: str, data) -> None:
+        """
+        Write file with support for content type and improved type handling.
+
+        :param path: Destination path in blob storage
+        :param data: File contents
+        """
+        blob_client = self.blob_service_client.get_blob_client(
+            container=self.container, blob=path, snapshot=None
+        )
+
+        if isinstance(data, str):
+            binary_data = data.encode()
+        elif isinstance(data, bytes):
+            binary_data = data
+        else:
+            raise Exception(f'Unsupported data type. Only "bytes" or "string" accepted')
+
+        blob_client.upload_blob(binary_data, overwrite=True)
+
+    def upload_file(self, file_path, blob_path):
+        """Uploads a single file to Azure Blob Storage."""
+        try:
+            blob_client = self.container_client.get_blob_client(blob_path)
+            with open(file_path, "rb") as data:
+                blob_client.upload_blob(data, overwrite=True)
+            print(f"Uploaded {file_path} to {blob_path}")
+        except Exception as e:
+            print(f"Failed to upload {file_path}: {e}")
+
+    def upload_directory(self, dir_path, blob_dir_path):
+        """Uploads all files from a directory to Azure Blob Storage."""
+        for root, dirs, files in os.walk(dir_path):
+            for file in files:
+                local_file_path = os.path.join(root, file)
+                relative_path = os.path.relpath(local_file_path, dir_path)
+                blob_file_path = os.path.join(blob_dir_path, relative_path).replace(
+                    "\\", "/"
+                )
+
+                self.upload_file(local_file_path, blob_file_path)
+
+    def download_directory(self, blob_dir_path: str, local_dir_path: str):
+        """Downloads all files from a directory in Azure Blob Storage to a local directory."""
+        try:
+            # Ensure the local directory exists
+            os.makedirs(local_dir_path, exist_ok=True)
+
+            # List all files in the blob directory
+            blob_items = self.container_client.list_blobs(
+                name_starts_with=blob_dir_path
+            )
+
+            for blob_item in blob_items:
+                # Get the relative path of the blob file
+                relative_path = os.path.relpath(blob_item.name, blob_dir_path)
+                # Construct the local file path
+                local_file_path = os.path.join(local_dir_path, relative_path)
+                # Create directories if needed
+                os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+                # Download the blob to the local file
+                blob_client = self.container_client.get_blob_client(blob_item.name)
+                with open(local_file_path, "wb") as file:
+                    file.write(blob_client.download_blob().readall())
+
+            print(f"Downloaded directory {blob_dir_path} to {local_dir_path}")
+        except Exception as e:
+            print(f"Failed to download directory {blob_dir_path}: {e}")
+
+    def copy_directory(self, source_dir: str, destination_dir: str):
+        """
+        Copies all files from a source directory to a destination directory within the same container.
+
+        :param source_dir: The source directory path in the blob storage
+        :param destination_dir: The destination directory path in the blob storage
+        """
+        try:
+            # Ensure source directory path ends with a trailing slash
+            source_dir = source_dir.rstrip("/") + "/"
+            destination_dir = destination_dir.rstrip("/") + "/"
+
+            # List all blobs in the source directory
+            source_blobs = self.container_client.list_blobs(name_starts_with=source_dir)
+
+            for blob in source_blobs:
+                # Get the relative path of the blob
+                relative_path = os.path.relpath(blob.name, source_dir)
+
+                # Construct the new blob path
+                new_blob_path = os.path.join(destination_dir, relative_path).replace(
+                    "\\", "/"
+                )
+
+                # Create a source blob client
+                source_blob_client = self.container_client.get_blob_client(blob.name)
+
+                # Create a destination blob client
+                destination_blob_client = self.container_client.get_blob_client(
+                    new_blob_path
+                )
+
+                # Start the copy operation
+                destination_blob_client.start_copy_from_url(source_blob_client.url)
+
+            print(f"Copied directory from {source_dir} to {destination_dir}")
+        except Exception as e:
+            print(f"Failed to copy directory {source_dir}: {e}")
+
+    def exists(self, path: str) -> bool:
+        blob_client = self.blob_service_client.get_blob_client(
+            container=self.container, blob=path, snapshot=None
+        )
+        return blob_client.exists()
+
+    def file_exists(self, path: str) -> bool:
+        return self.exists(path) and not self.is_dir(path)
+
+    def file_size(self, path: str) -> float:
+        blob_client = self.blob_service_client.get_blob_client(
+            container=self.container, blob=path, snapshot=None
+        )
+        properties = blob_client.get_blob_properties()
+
+        # The size is in bytes, convert it to kilobytes
+        size_in_bytes = properties.size
+        size_in_kb = size_in_bytes / 1024.0
+        return size_in_kb
+
+    def list_files(self, path: str):
+        blob_items = self.container_client.list_blobs(name_starts_with=path)
+        return [item["name"] for item in blob_items]
+
+    def walk(self, top: str):
+        top = top.rstrip("/") + "/"
+        blob_items = self.container_client.list_blobs(name_starts_with=top)
+        blobs = [item["name"] for item in blob_items]
+        for blob in blobs:
+            dirpath, filename = os.path.split(blob)
+            yield (dirpath, [], [filename])
+
+    def list_directories(self, path: str) -> list:
+        """List only directory names (not files) from a given path in ADLS."""
+        search_path = path.rstrip("/") + "/" if path else ""
+
+        blob_items = self.container_client.list_blobs(name_starts_with=search_path)
+
+        directories = set()
+
+        for blob_item in blob_items:
+            # Get the relative path from the search path
+            relative_path = blob_item.name[len(search_path) :]
+
+            # Skip if it's empty (shouldn't happen but just in case)
+            if not relative_path:
+                continue
+
+            # If there's a "/" in the relative path, it means there's a subdirectory
+            if "/" in relative_path:
+                # Get the first directory name
+                dir_name = relative_path.split("/")[0]
+                directories.add(dir_name)
+
+        return sorted(list(directories))
+
+    @contextlib.contextmanager
+    def open(self, path: str, mode: str = "r"):
+        """
+        Context manager for file operations with enhanced mode support.
+
+        :param path: File path in blob storage
+        :param mode: File open mode (r, rb, w, wb)
+        """
+        if mode == "w":
+            file = io.StringIO()
+            yield file
+            self.write_file(path, file.getvalue())
+
+        elif mode == "wb":
+            file = io.BytesIO()
+            yield file
+            self.write_file(path, file.getvalue())
+
+        elif mode == "r":
+            data = self.read_file(path, encoding="UTF-8")
+            file = io.StringIO(data)
+            yield file
+
+        elif mode == "rb":
+            data = self.read_file(path)
+            file = io.BytesIO(data)
+            yield file
+
+    def get_file_metadata(self, path: str) -> dict:
+        """
+        Retrieve comprehensive file metadata.
+
+        :param path: File path in blob storage
+        :return: File metadata dictionary
+        """
+        blob_client = self.container_client.get_blob_client(path)
+        properties = blob_client.get_blob_properties()
+
+        return {
+            "name": path,
+            "size_bytes": properties.size,
+            "content_type": properties.content_settings.content_type,
+            "last_modified": properties.last_modified,
+            "etag": properties.etag,
+        }
+
+    def is_file(self, path: str) -> bool:
+        return self.file_exists(path)
+
+    def is_dir(self, path: str) -> bool:
+        dir_path = path.rstrip("/") + "/"
+
+        existing_blobs = self.list_files(dir_path)
+
+        if len(existing_blobs) > 1:
+            return True
+        elif len(existing_blobs) == 1:
+            if existing_blobs[0] != path.rstrip("/"):
+                return True
+
+        return False
+
+    def rmdir(self, dir: str) -> None:
+        blobs = self.list_files(dir)
+        self.container_client.delete_blobs(*blobs)
+
+    def mkdir(self, path: str, exist_ok: bool = False) -> None:
+        """
+        Create a directory in Azure Blob Storage.
+
+        In ADLS, directories are conceptual and created by adding a placeholder blob.
+
+        :param path: Path of the directory to create
+        :param exist_ok: If False, raise an error if the directory already exists
+        """
+        dir_path = path.rstrip("/") + "/"
+
+        existing_blobs = list(self.list_files(dir_path))
+
+        if existing_blobs and not exist_ok:
+            raise FileExistsError(f"Directory {path} already exists")
+
+        # Create a placeholder blob to represent the directory
+        placeholder_blob_path = os.path.join(dir_path, ".placeholder")
+
+        # Only create placeholder if it doesn't already exist
+        if not self.file_exists(placeholder_blob_path):
+            placeholder_content = (
+                b"This is a placeholder blob to represent a directory."
+            )
+            blob_client = self.blob_service_client.get_blob_client(
+                container=self.container, blob=placeholder_blob_path
+            )
+            blob_client.upload_blob(placeholder_content, overwrite=True)
+
+    def remove(self, path: str) -> None:
+        blob_client = self.blob_service_client.get_blob_client(
+            container=self.container, blob=path, snapshot=None
+        )
+        if blob_client.exists():
+            blob_client.delete_blob()

gigaspatial/core/io/data_api.py

@@ -0,0 +1,113 @@
+import pandas as pd
+import delta_sharing
+from typing import Union
+from pathlib import Path
+
+from gigaspatial.config import config
+
+
+class GigaDataAPI:
+
+    def __init__(
+        self,
+        profile_file: Union[str, Path] = config.API_PROFILE_FILE_PATH,
+        share_name: str = config.API_SHARE_NAME,
+        schema_name: str = config.API_SCHEMA_NAME,
+    ):
+        """
+        Initialize the GigaDataAPI class with the profile file, share name, and schema name.
+
+        profile_file: Path to the delta-sharing profile file.
+        share_name: Name of the share (e.g., "gold").
+        schema_name: Name of the schema (e.g., "school-master").
+        """
+        self.profile_file = profile_file
+        self.share_name = share_name
+        self.schema_name = schema_name
+        self.client = delta_sharing.SharingClient(profile_file)
+
+        self._cache = {}
+
+    def get_country_list(self, sort=True):
+        """
+        Retrieve a list of available countries in the dataset.
+
+        :param sort: Whether to sort the country list alphabetically (default is True).
+        """
+        country_list = [
+            t.name for t in self.client.list_all_tables() if t.share == self.share_name
+        ]
+        if sort:
+            country_list.sort()
+        return country_list
+
+    def load_country_data(self, country, filters=None, use_cache=True):
+        """
+        Load the dataset for the specified country with optional filtering and caching.
+
+        country: The country code (e.g., "MWI").
+        filters: A dictionary with column names as keys and filter values as values.
+        use_cache: Whether to use cached data if available (default is True).
+        """
+        # Check if data is cached
+        if use_cache and country in self._cache:
+            df_country = self._cache[country]
+        else:
+            # Load data from the API
+            table_url = (
+                f"{self.profile_file}#{self.share_name}.{self.schema_name}.{country}"
+            )
+            df_country = delta_sharing.load_as_pandas(table_url)
+            self._cache[country] = df_country  # Cache the data
+
+        # Apply filters if provided
+        if filters:
+            for column, value in filters.items():
+                df_country = df_country[df_country[column] == value]
+
+        return df_country
+
+    def load_multiple_countries(self, countries):
+        """
+        Load data for multiple countries and combine them into a single DataFrame.
+
+        countries: A list of country codes.
+        """
+        df_list = []
+        for country in countries:
+            df_list.append(self.load_country_data(country))
+        return pd.concat(df_list, ignore_index=True)
+
+    def get_country_metadata(self, country):
+        """
+        Retrieve metadata (e.g., column names and data types) for a country's dataset.
+
+        country: The country code (e.g., "MWI").
+        """
+        df_country = self.load_country_data(country)
+        metadata = {
+            "columns": df_country.columns.tolist(),
+            "data_types": df_country.dtypes.to_dict(),
+            "num_records": len(df_country),
+        }
+        return metadata
+
+    def get_all_cached_data_as_dict(self):
+        """
+        Retrieve all cached data in a dictionary format, where each key is a country code,
+        and the value is the DataFrame of that country.
+        """
+        return self._cache if self._cache else {}
+
+    def get_all_cached_data_as_json(self):
+        """
+        Retrieve all cached data in a JSON-like format. Each country is represented as a key,
+        and the value is a list of records (i.e., the DataFrame's `to_dict(orient='records')` format).
+        """
+        if not self._cache:
+            return {}
+
+        # Convert each DataFrame in the cache to a JSON-like format (list of records)
+        return {
+            country: df.to_dict(orient="records") for country, df in self._cache.items()
+        }