giga-spatial 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl

gigaspatial/processing/tif_processor.py

@@ -1,7 +1,7 @@
 import numpy as np
 import pandas as pd
 import geopandas as gpd
-from typing import List, Optional, Tuple, Union, Literal
+from typing import List, Optional, Tuple, Union, Literal, Callable
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
 from contextlib import contextmanager
@@ -9,6 +9,9 @@ from shapely.geometry import box, Polygon, MultiPolygon
 from pathlib import Path
 import rasterio
 from rasterio.mask import mask
+from functools import partial
+import multiprocessing
+from tqdm import tqdm
 
 from gigaspatial.core.io.data_store import DataStore
 from gigaspatial.core.io.local_data_store import LocalDataStore
@@ -113,6 +116,9 @@ class TifProcessor:
     @property
     def tabular(self) -> pd.DataFrame:
         """Get the data from the TIF file"""
+        self.logger.warning(
+            "The `tabular` property is deprecated, use `to_dataframe` instead"
+        )
         if not hasattr(self, "_tabular"):
             try:
                 if self.mode == "single":
@@ -142,14 +148,56 @@ class TifProcessor:
 
         return self._tabular
 
-    def to_dataframe(self) -> pd.DataFrame:
-        return self.tabular
+    def to_dataframe(self, drop_nodata=True, **kwargs) -> pd.DataFrame:
+        try:
+            if self.mode == "single":
+                df = self._to_band_dataframe(drop_nodata=drop_nodata, **kwargs)
+            elif self.mode == "rgb":
+                df = self._to_rgb_dataframe(drop_nodata=drop_nodata)
+            elif self.mode == "rgba":
+                df = self._to_rgba_dataframe(drop_transparent=drop_nodata)
+            elif self.mode == "multi":
+                df = self._to_multi_band_dataframe(drop_nodata=drop_nodata, **kwargs)
+            else:
+                raise ValueError(
+                    f"Invalid mode: {self.mode}. Must be one of: single, rgb, rgba, multi"
+                )
+        except Exception as e:
+            raise ValueError(
+                f"Failed to process TIF file in mode '{self.mode}'. "
+                f"Please ensure the file is valid and matches the selected mode. "
+                f"Original error: {str(e)}"
+            )
+
+        return df
+
+    def to_geodataframe(self, **kwargs) -> gpd.GeoDataFrame:
+        """
+        Convert the processed TIF data into a GeoDataFrame, where each row represents a pixel zone.
+        Each zone is defined by its bounding box, based on pixel resolution and coordinates.
+        """
+        df = self.to_dataframe(**kwargs)
+
+        x_res, y_res = self.resolution
+
+        # create bounding box for each pixel
+        geometries = [
+            box(lon - x_res / 2, lat - y_res / 2, lon + x_res / 2, lat + y_res / 2)
+            for lon, lat in zip(df["lon"], df["lat"])
+        ]
+
+        gdf = gpd.GeoDataFrame(df, geometry=geometries, crs=self.crs)
+
+        return gdf
 
     def get_zoned_geodataframe(self) -> gpd.GeoDataFrame:
         """
         Convert the processed TIF data into a GeoDataFrame, where each row represents a pixel zone.
         Each zone is defined by its bounding box, based on pixel resolution and coordinates.
         """
+        self.logger.warning(
+            "The `get_zoned_geodataframe` method is deprecated, use `to_geodataframe` instead"
+        )
         self.logger.info("Converting data to GeoDataFrame with zones...")
 
         df = self.tabular
@@ -168,7 +216,7 @@ class TifProcessor:
         return gdf
 
     def sample_by_coordinates(
-        self, coordinate_list: List[Tuple[float, float]]
+        self, coordinate_list: List[Tuple[float, float]], **kwargs
     ) -> Union[np.ndarray, dict]:
         self.logger.info("Sampling raster values at the coordinates...")
 
@@ -200,63 +248,188 @@ class TifProcessor:
                     ]
 
                 return rgb_values
+            elif self.count > 1:
+                return np.array(
+                    [vals for vals in src.sample(coordinate_list, **kwargs)]
+                )
             else:
-                if src.count != 1:
-                    raise ValueError("Single band mode requires a 1-band TIF file")
                 return np.array([vals[0] for vals in src.sample(coordinate_list)])
 
     def sample_by_polygons(
-        self, polygon_list: List[Union[Polygon, MultiPolygon]], stat: str = "mean"
-    ) -> np.ndarray:
+        self,
+        polygon_list,
+        stat: Union[str, Callable, List[Union[str, Callable]]] = "mean",
+    ):
         """
-        Sample raster values within each polygon of a GeoDataFrame.
+        Sample raster values by polygons and compute statistic(s) for each polygon.
+
+        Args:
+            polygon_list: List of shapely Polygon or MultiPolygon objects.
+            stat: Statistic(s) to compute. Can be:
+                - Single string: 'mean', 'median', 'sum', 'min', 'max', 'std', 'count'
+                - Single callable: custom function that takes array and returns scalar
+                - List of strings/callables: multiple statistics to compute
 
-        Parameters:
-            polygon_list: List of polygon geometries (can include MultiPolygons).
-            stat (str): Aggregation statistic to compute within each polygon.
-                        Options: "mean", "median", "sum", "min", "max".
         Returns:
-            A NumPy array of sampled values
+            If single stat: np.ndarray of computed statistics for each polygon
+            If multiple stats: List of dictionaries with stat names as keys
         """
-        self.logger.info("Sampling raster values within polygons...")
-
-        with self.open_dataset() as src:
-            results = []
+        # Determine if single or multiple stats
+        single_stat = not isinstance(stat, list)
+        stats_list = [stat] if single_stat else stat
+
+        # Prepare stat functions
+        stat_funcs = []
+        stat_names = []
+
+        for s in stats_list:
+            if callable(s):
+                stat_funcs.append(s)
+                stat_names.append(
+                    s.__name__
+                    if hasattr(s, "__name__")
+                    else f"custom_{len(stat_names)}"
+                )
+            else:
+                # Handle string statistics
+                if s == "count":
+                    stat_funcs.append(len)
+                else:
+                    stat_funcs.append(getattr(np, s))
+                stat_names.append(s)
 
-            for geom in polygon_list:
-                if geom.is_empty:
-                    results.append(np.nan)
-                    continue
+        results = []
 
+        with self.open_dataset() as src:
+            for polygon in tqdm(polygon_list):
                 try:
-                    # Mask the raster with the polygon
-                    out_image, _ = mask(src, [geom], crop=True)
-
-                    # Flatten the raster values and remove NoData values
-                    values = out_image[out_image != src.nodata].flatten()
+                    out_image, _ = mask(src, [polygon], crop=True, filled=False)
 
-                    # Compute the desired statistic
-                    if len(values) == 0:
-                        results.append(np.nan)
+                    # Use masked arrays for more efficient nodata handling
+                    if hasattr(out_image, "mask"):
+                        valid_data = out_image.compressed()
+                    else:
+                        valid_data = (
+                            out_image[out_image != self.nodata]
+                            if self.nodata
+                            else out_image.flatten()
+                        )
+
+                    if len(valid_data) == 0:
+                        if single_stat:
+                            results.append(np.nan)
+                        else:
+                            results.append({name: np.nan for name in stat_names})
                     else:
-                        if stat == "mean":
-                            results.append(np.mean(values))
-                        elif stat == "median":
-                            results.append(np.median(values))
-                        elif stat == "sum":
-                            results.append(np.sum(values))
-                        elif stat == "min":
-                            results.append(np.min(values))
-                        elif stat == "max":
-                            results.append(np.max(values))
+                        if single_stat:
+                            results.append(stat_funcs[0](valid_data))
                         else:
-                            raise ValueError(f"Unknown statistic: {stat}")
+                            # Compute all statistics for this polygon
+                            polygon_stats = {}
+                            for func, name in zip(stat_funcs, stat_names):
+                                try:
+                                    polygon_stats[name] = func(valid_data)
+                                except Exception:
+                                    polygon_stats[name] = np.nan
+                            results.append(polygon_stats)
+
+                except Exception:
+                    if single_stat:
+                        results.append(np.nan)
+                    else:
+                        results.append({name: np.nan for name in stat_names})
+
+        return np.array(results) if single_stat else results
+
+    def sample_by_polygons_batched(
+        self,
+        polygon_list: List[Union[Polygon, MultiPolygon]],
+        stat: Union[str, Callable] = "mean",
+        batch_size: int = 100,
+        n_workers: int = 4,
+        **kwargs,
+    ) -> np.ndarray:
+        """
+        Sample raster values by polygons in parallel using batching.
+        """
+
+        def _chunk_list(data_list, chunk_size):
+            """Yield successive chunks from data_list."""
+            for i in range(0, len(data_list), chunk_size):
+                yield data_list[i : i + chunk_size]
+
+        if len(polygon_list) == 0:
+            return np.array([])
+
+        stat_func = stat if callable(stat) else getattr(np, stat)
+
+        polygon_chunks = list(_chunk_list(polygon_list, batch_size))
+
+        with multiprocessing.Pool(
+            initializer=self._initializer_worker, processes=n_workers
+        ) as pool:
+            process_func = partial(self._process_polygon_batch, stat_func=stat_func)
+            batched_results = list(
+                tqdm(
+                    pool.imap(process_func, polygon_chunks),
+                    total=len(polygon_chunks),
+                    desc=f"Sampling polygons",
+                )
+            )
 
-                except Exception as e:
-                    self.logger.error(f"Error processing polygon: {e}")
-                    results.append(np.nan)
+            results = [item for sublist in batched_results for item in sublist]
 
         return np.array(results)
+    
+    def _initializer_worker(self):
+        """
+        Initializer function for each worker process.
+        Opens the raster dataset and stores it in a process-local variable.
+        This function runs once per worker, not for every task.
+        """
+        global src_handle
+        with self.data_store.open(self.dataset_path, "rb") as f:
+            with rasterio.MemoryFile(f.read()) as memfile:
+                src_handle = memfile.open()
+
+    def _process_single_polygon(self, polygon, stat_func):
+        """
+        Helper function to process a single polygon.
+        This will be run in a separate process.
+        """
+        global src_handle
+        if src_handle is None:
+            # This should not happen if the initializer is set up correctly,
+            # but it's a good defensive check.
+            raise RuntimeError("Raster dataset not initialized in this process.")
+
+        try:
+            out_image, _ = mask(src_handle, [polygon], crop=True, filled=False)
+
+            if hasattr(out_image, "mask"):
+                valid_data = out_image.compressed()
+            else:
+                valid_data = (
+                    out_image[out_image != self.nodata]
+                    if self.nodata
+                    else out_image.flatten()
+                )
+
+            if len(valid_data) == 0:
+                return np.nan
+            else:
+                return stat_func(valid_data)
+        except Exception:
+            return np.nan
+
+    def _process_polygon_batch(self, polygon_batch, stat_func):
+        """
+        Processes a batch of polygons.
+        """
+        return [
+            self._process_single_polygon(polygon, stat_func)
+            for polygon in polygon_batch
+        ]
 
     def _to_rgba_dataframe(self, drop_transparent: bool = False) -> pd.DataFrame:
         """
@@ -554,7 +727,9 @@ def sample_multiple_tifs_by_polygons(
     sampled_values = np.full(len(polygon_list), np.nan, dtype=np.float32)
 
     for tp in tif_processors:
-        values = tp.sample_by_polygons(polygon_list=polygon_list, stat=stat)
+        values = tp.sample_by_polygons(
+            polygon_list=polygon_list, stat=stat
+        )
 
         mask = np.isnan(sampled_values)  # replace all NaNs
 

giga_spatial-0.6.3.dist-info/RECORD

@@ -1,47 +0,0 @@
-giga_spatial-0.6.3.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-gigaspatial/__init__.py,sha256=zYiFHqR7JwbvdK9dvKrh-RTNfUqjHUwC4CTcFAPVYLc,22
-gigaspatial/config.py,sha256=yMf1ofOU0_I6iKDqshiFSYmK6TDIVpPm1AZo4e2okHU,8166
-gigaspatial/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-gigaspatial/core/io/__init__.py,sha256=y4QNWx6US1-adTuAO_NZwLmjzSQj25HNDL5hUGvEHZc,263
-gigaspatial/core/io/adls_data_store.py,sha256=Zv-D_8d_2h57HnCUTJb0JWWjXqR_0XH4F8Nu_UFZK9E,11975
-gigaspatial/core/io/data_api.py,sha256=3HMstau3zH3JPRUW0t83DZt74N39bt-jsfAyrUUFMoc,3944
-gigaspatial/core/io/data_store.py,sha256=mi8fy78Dtwj4dpKkyDM6kTlna1lfCQ5ro2hUAOFr83A,3223
-gigaspatial/core/io/local_data_store.py,sha256=hcu7DNYa3AL6sEPMqguzxWal_bnP7CIpbwpoiyf5TCw,2933
-gigaspatial/core/io/readers.py,sha256=gqFKGRCsAP_EBXipqGtT8MEV-x0u6SrCqaSiOC5YPTA,9284
-gigaspatial/core/io/writers.py,sha256=asb56ZHQEWO2rdilIq7QywDRk8yfebecWv1KwzUpaXI,4367
-gigaspatial/core/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-gigaspatial/core/schemas/entity.py,sha256=QAhEW0-JgdWh9pjKGbB5ArvqtVK85ayYZJPgjdb-IKw,8590
-gigaspatial/generators/__init__.py,sha256=dGxmWHwkvjYopdlE9d4tEaQtn4uh391zZ7NhOG5OVTk,153
-gigaspatial/generators/poi.py,sha256=meP_bnGnKout8ik1G2XSCyKM0RhegCZg-5woVXLqIog,26509
-gigaspatial/generators/zonal/__init__.py,sha256=6bDVRRxim5qpdTE_b2dLeWmrTrDSZBnM7gn5MztCbGk,233
-gigaspatial/generators/zonal/base.py,sha256=up3k5ApPn9387DRu34VIJqA1_6lWdi5UpGVjcQQIDsE,15346
-gigaspatial/generators/zonal/geometry.py,sha256=XPcX5lT7X7Z1vn72sN-VKLb2hDP9F_w3LwdRNecmJn0,18339
-gigaspatial/generators/zonal/mercator.py,sha256=R_KlaqF4lnc0cRqVfcNVO8i0Re21_6w7pnclVKSohcY,3125
-gigaspatial/grid/__init__.py,sha256=H8SnNAMDafJXJ9bUp2zU0Z3t6s8niqY5rGP5nFhnbLA,45
-gigaspatial/grid/mercator_tiles.py,sha256=Z_3M4sy1tyxywAo2wmBb6niBP3x-IWgwMkmUp8LOSDg,10492
-gigaspatial/handlers/__init__.py,sha256=pqK3rJtelOAkBaWNhpGy2t-p_zrwO-9BqABLQufTXF0,1449
-gigaspatial/handlers/base.py,sha256=rL94c3wDjsqzLp4na8FfYXW6tNjVGX6v4M-Ce4LrAro,26413
-gigaspatial/handlers/boundaries.py,sha256=hoO-b5MlFYwlCWogApcFyEx6OnxMJG29lqJurNGwOWg,11260
-gigaspatial/handlers/ghsl.py,sha256=GHao8lkmj1C0-QFqNwH9jr0Lqzu6NTj_7ooQdj1h6ok,27760
-gigaspatial/handlers/giga.py,sha256=2aP1EenDAQXn-h-uCyuVxEVZvAFEvrL17_z0MiS8FDs,4867
-gigaspatial/handlers/google_open_buildings.py,sha256=Liqk7qJhDtB4Ia4uhBe44LFcf-XVKBjRfj-pWlE5erY,16594
-gigaspatial/handlers/hdx.py,sha256=DNw-LhxuJU3eNGihQGyPJT0a1PaOCupNHr7BDGal4Zo,18088
-gigaspatial/handlers/mapbox_image.py,sha256=M_nkJ_b1PD8FG1ajVgSycCb0NRTAI_SLpHdzszNetKA,7786
-gigaspatial/handlers/maxar_image.py,sha256=g5YVGV-8JjeG9bGBOp7ZfKani22J4izXX4hnB9A99Jk,10272
-gigaspatial/handlers/microsoft_global_buildings.py,sha256=bQ5WHIv3v0wWrZZUbZkKPRjgdlqIxlK7CV_0zSvdrTw,20292
-gigaspatial/handlers/ookla_speedtest.py,sha256=EcvSAxJZ9GPfzYnT_C85Qgy2ecc9ndf70Pklk53OdC8,6506
-gigaspatial/handlers/opencellid.py,sha256=KuJqd-5-RO5ZzyDaBSrTgCK2ib5N_m3RUcPlX5heWwI,10683
-gigaspatial/handlers/osm.py,sha256=sLNMkOVh1v50jrWw7Z0-HILY5QTQjgKCHCeAfXj5jA8,14084
-gigaspatial/handlers/overture.py,sha256=lKeNw00v5Qia7LdWORuYihnlKEqxE9m38tdeRrvag9k,4218
-gigaspatial/handlers/rwi.py,sha256=GDpQH9K96QZD3yezJOBiy5yZvYmrj4xbjUNSjYfNAh0,4875
-gigaspatial/handlers/unicef_georepo.py,sha256=ODYNvkU_UKgOHXT--0MqmJ4Uk6U1_mp9xgehbTzKpX8,31924
-gigaspatial/handlers/worldpop.py,sha256=oJ39NGajXi0rn829ZoFiaeG4_wavyPvljdActpxs12I,9850
-gigaspatial/processing/__init__.py,sha256=QDVL-QbLCrIb19lrajP7LrHNdGdnsLeGcvAs_jQpdRM,183
-gigaspatial/processing/geo.py,sha256=D-S3IlhQwLIxrCcxy6NhNmKLrOIjoRHfK_eZJGKpe2U,36947
-gigaspatial/processing/sat_images.py,sha256=YUbH5MFNzl6NX49Obk14WaFcr1s3SyGJIOk-kRpbBNg,1429
-gigaspatial/processing/tif_processor.py,sha256=zqcP_ioo9KHNJ6H0uba4UghW4MToTRwq1iE-nZbb8zA,21101
-gigaspatial/processing/utils.py,sha256=HC85vGKQakxlkoQAkZmeAXWHsenAwTIRn7jPKUA7x20,1500
-giga_spatial-0.6.3.dist-info/METADATA,sha256=Aw5adPdTcA3AuJBmZgAG4rJQYW4dJqw2GT90mYE7cgU,7467
-giga_spatial-0.6.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-giga_spatial-0.6.3.dist-info/top_level.txt,sha256=LZsccgw6H4zXT7m6Y4XChm-Y5LjHAwZ2hkGN_B3ExmI,12
-giga_spatial-0.6.3.dist-info/RECORD,,