geo-explorer 0.9.10__py3-none-any.whl → 0.9.12__py3-none-any.whl

geo_explorer/file_browser.py

@@ -19,6 +19,9 @@ from .utils import _clicked_button_style
 from .utils import _standardize_path
 from .utils import _unclicked_button_style
 from .utils import get_button_with_tooltip
+from .utils import time_function_call
+from .utils import time_method_call
+from .utils import _PROFILE_DICT
 
 
 class FileBrowser:
@@ -29,15 +32,18 @@ class FileBrowser:
         start_dir: str,
         favorites: list[str] | None = None,
         file_system: AbstractFileSystem | None = None,
+        sum_partition_sizes: bool = True,
     ) -> None:
         self.start_dir = _standardize_path(start_dir)
         self.file_system = file_system
         self.favorites = (
             [_standardize_path(x) for x in favorites] if favorites is not None else []
         )
+        self.sum_partition_sizes = sum_partition_sizes
         self._history = [self.start_dir]
         self._register_callbacks()
 
+    @time_method_call(_PROFILE_DICT)
     def get_file_browser_components(self, width: str = "140vh") -> list[Component]:
         return [
             dbc.Row(
@@ -295,6 +301,7 @@ class FileBrowser:
             State("file-list", "children"),
             State("file-data-dict", "data"),
         )
+        @time_method_call(_PROFILE_DICT)
         def update_file_list(
             path,
             search_word,
@@ -350,6 +357,7 @@ class FileBrowser:
 
             return (file_data_dict, file_list, alert, sort_by_clicks, self._history[1:])
 
+    @time_method_call(_PROFILE_DICT)
     def _list_dir(
         self,
         path: str,
@@ -386,11 +394,13 @@ class FileBrowser:
 
         if (recursive or 0) % 2 == 0:
 
+            @time_function_call(_PROFILE_DICT)
             def _ls(path):
                 return file_system.ls(path, detail=True)
 
         else:
 
+            @time_function_call(_PROFILE_DICT)
             def _ls(path):
                 path = str(Path(path) / "**")
                 return _try_glob(path, file_system)
@@ -417,7 +427,7 @@ class FileBrowser:
         if isinstance(paths, dict):
             paths = list(paths.values())
 
-        def is_dir_or_is_partitioned_parquet(x) -> bool:
+        def is_dir_or_relevant_file_format(x) -> bool:
             return x["type"] == "directory" or any(
                 x["name"].endswith(txt) for txt in self.file_formats
             )
@@ -427,13 +437,35 @@ class FileBrowser:
             for x in paths
             if isinstance(x, dict)
             and _contains(x["name"])
-            and is_dir_or_is_partitioned_parquet(x)
+            and is_dir_or_relevant_file_format(x)
             and Path(path).parts != Path(x["name"]).parts
         ]
 
         paths.sort(key=lambda x: x["name"])
         isdir_list = [x["type"] == "directory" for x in paths]
 
+        if self.sum_partition_sizes:
+            self._sum_partition_sizes(paths, file_system)
+
+        return (
+            paths,
+            [
+                _get_file_list_row(
+                    x["name"],
+                    x.get("updated", None),
+                    x["size"],
+                    isdir,
+                    path,
+                    self.file_formats,
+                    file_system,
+                )
+                for x, isdir in zip(paths, isdir_list, strict=True)
+                if isinstance(x, dict)
+            ],
+            None,
+        )
+
+    def _sum_partition_sizes(self, paths, file_system):
         partitioned = {
             i: x
             for i, x in enumerate(paths)
@@ -444,6 +476,7 @@ class FileBrowser:
             )
         }
 
+        @time_function_call(_PROFILE_DICT)
         def get_summed_size_and_latest_timestamp_in_subdirs(
             x,
         ) -> tuple[float, datetime.datetime]:
@@ -462,11 +495,9 @@ class FileBrowser:
             )
 
         with ThreadPoolExecutor() as executor:
-            summed_size_ant_time = list(
-                executor.map(
-                    get_summed_size_and_latest_timestamp_in_subdirs,
-                    partitioned.values(),
-                )
+            summed_size_ant_time = executor.map(
+                get_summed_size_and_latest_timestamp_in_subdirs,
+                partitioned.values(),
             )
             for i, (size, timestamp) in zip(
                 partitioned, summed_size_ant_time, strict=True
@@ -474,25 +505,8 @@ class FileBrowser:
                 paths[i]["size"] = size
                 paths[i]["updated"] = timestamp
 
-        return (
-            paths,
-            [
-                _get_file_list_row(
-                    x["name"],
-                    x.get("updated", None),
-                    x["size"],
-                    isdir,
-                    path,
-                    self.file_formats,
-                    file_system,
-                )
-                for x, isdir in zip(paths, isdir_list, strict=True)
-                if isinstance(x, dict)
-            ],
-            None,
-        )
-
 
+@time_function_call(_PROFILE_DICT)
 def _get_file_list_row(
     path, timestamp, size, isdir: bool, current_path, file_formats, file_system
 ):
@@ -564,6 +578,7 @@ def _get_file_list_row(
     )
 
 
+@time_function_call(_PROFILE_DICT)
 def _try_glob(path, file_system):
     try:
         return file_system.glob(path, detail=True, recursive=True)

geo_explorer/geo_explorer.py

@@ -25,6 +25,7 @@ from typing import Any
 from typing import ClassVar
 
 import dash
+import pyproj
 import dash_bootstrap_components as dbc
 import dash_leaflet as dl
 import folium
@@ -39,7 +40,6 @@ import pandas as pd
 import polars as pl
 import pyarrow
 import pyarrow.parquet as pq
-import rasterio
 import sgis as sg
 import shapely
 from dash import Dash
@@ -82,9 +82,9 @@ except ImportError:
 
 from .file_browser import FileBrowser
 from .fs import LocalFileSystem
-from .nc import GeoTIFFConfig
-from .nc import NetCDFConfig
-from .nc import _run_code_block
+from .img import GeoTIFFConfig
+from .img import AbstractImageConfig
+from .img import NetCDFConfig
 from .utils import _PROFILE_DICT
 from .utils import DEBUG
 from .utils import _clicked_button_style
@@ -687,12 +687,15 @@ def _read_files(explorer, paths: list[str], mask=None, **kwargs) -> None:
     paths = [
         path
         for path in paths
-        if mask is None
-        or (
-            path in bbox_set
-            and (
-                pd.isna(explorer._bbox_series[path])
-                or shapely.intersects(mask, explorer._bbox_series[path])
+        if explorer._loaded_data_sizes[path] > 0
+        and (
+            mask is None
+            or (
+                path in bbox_set
+                and (
+                    pd.isna(explorer._bbox_series[path])
+                    or shapely.intersects(mask, explorer._bbox_series[path])
+                )
             )
         )
     ]
@@ -769,30 +772,55 @@ def _try_to_get_bbox_else_none(
 
 
 def _get_bbox_series_as_4326(paths, file_system):
-    # bbox_series = sg.get_bbox_series(paths, file_system=file_system)
-    # return bbox_series.to_crs(4326)
-
     func = partial(_try_to_get_bbox_else_none, file_system=file_system)
     with ThreadPoolExecutor() as executor:
         bbox_and_crs = list(executor.map(func, paths))
 
-    crss = {json.dumps(x[1]) for x in bbox_and_crs}
-    crss = {
-        crs
-        for crs in crss
-        if not any(str(crs).lower() == txt for txt in ["none", "null"])
-    }
+    bbox_and_crs = [(bbox, json.dumps(crs)) for bbox, crs in bbox_and_crs]
+    bbox_and_crs = [
+        (
+            bbox,
+            (
+                pyproj.CRS(crs).to_string()
+                if not any(crs.lower() == txt for txt in ["none", "null"])
+                else None
+            ),
+        )
+        for bbox, crs in bbox_and_crs
+    ]
+    crss = {crs for (_, crs) in bbox_and_crs}
     if not crss:
         return GeoSeries([None for _ in range(len(paths))], index=paths)
-    crs = get_common_crs(crss)
-    return GeoSeries(
-        [
-            shapely.box(*bbox[0]) if bbox[0] is not None else None
-            for bbox in bbox_and_crs
-        ],
-        index=paths,
-        crs=crs,
-    ).to_crs(4326)
+    missing = GeoSeries(
+        {
+            path: Polygon()
+            for path, (bbox, _) in zip(paths, bbox_and_crs, strict=True)
+            if bbox is None
+        }
+    )
+    paths_bbox_and_crs = {
+        path: (bbox, crs)
+        for path, (bbox, crs) in zip(paths, bbox_and_crs, strict=True)
+        if bbox is not None
+    }
+    crs_with_paths = {}
+    geoms = shapely.box(
+        [bbox[0] for (bbox, _) in paths_bbox_and_crs.values()],
+        [bbox[1] for (bbox, _) in paths_bbox_and_crs.values()],
+        [bbox[2] for (bbox, _) in paths_bbox_and_crs.values()],
+        [bbox[3] for (bbox, _) in paths_bbox_and_crs.values()],
+    )
+    for crs in crss:
+        crs_with_paths[crs] = {
+            path: geoms[i]
+            for i, (path, (bbox, this_crs)) in enumerate(paths_bbox_and_crs.items())
+            if this_crs == crs and bbox is not None
+        }
+    no_crs = GeoSeries(crs_with_paths.pop(None, {}), crs=4326)
+    return pd.concat(
+        [GeoSeries(data, crs=crs).to_crs(4326) for crs, data in crs_with_paths.items()]
+        + [missing, no_crs]
+    )
 
 
 def get_index(values: list[Any], ids: list[Any], index: Any):
@@ -1257,7 +1285,9 @@ class GeoExplorer:
         port: int = 8050,
         file_system: AbstractFileSystem | None = None,
         data: (
-            dict[str, str | GeoDataFrame | NetCDFConfig] | list[str | dict] | None
+            dict[str, str | GeoDataFrame | AbstractImageConfig]
+            | list[str | dict]
+            | None
         ) = None,
         column: str | None = None,
         color_dict: dict | None = None,
@@ -1273,6 +1303,7 @@ class GeoExplorer:
         nan_color: str = "#969696",
         nan_label: str = "Missing",
         max_read_size_per_callback: int = 1e9,
+        sum_partition_sizes: bool = True,
         **kwargs,
     ) -> None:
         """Initialiser."""
@@ -1313,7 +1344,10 @@ class GeoExplorer:
         self._concatted_data: pl.DataFrame | None = None
         self._selected_features = {}
         self._file_browser = FileBrowser(
-            start_dir, file_system=file_system, favorites=favorites
+            start_dir,
+            file_system=file_system,
+            favorites=favorites,
+            sum_partition_sizes=sum_partition_sizes,
         )
         self._current_table_view = None
         self.max_read_size_per_callback = max_read_size_per_callback
@@ -1718,17 +1752,25 @@ class GeoExplorer:
                 raise ValueError(error_mess)
             for key, value in x.items():
                 key = _standardize_path(key)
-                if isinstance(value, NetCDFConfig):
-                    # setting nc files as unchecked because they might be very large
-                    self.selected_files[key] = False
-                    self._queries[key] = value.code_block
+                if isinstance(value, AbstractImageConfig):
                     self._nc[key] = value
+                    # setting image files as unchecked because they might be very large
+                    self.selected_files[key] = False
+                    self.set_query(key, value.code_block)
+                    child_paths = self._get_child_paths([key]) or [key]
+                    try:
+                        bbox_series_dict |= {
+                            x: shapely.box(*self._nc[key].get_bounds(None, x))
+                            for x in child_paths
+                        }
+                    except Exception:
+                        pass
                     continue
                 if value is not None and not isinstance(value, (GeoDataFrame | str)):
                     raise ValueError(error_mess)
                 elif not isinstance(value, GeoDataFrame):
                     self.selected_files[key] = True
-                    self._queries[key] = value
+                    self.set_query(key, value)
                     continue
                 value, dtypes = _geopandas_to_polars(value, key)
                 bbox_series_dict[key] = shapely.box(
@@ -1785,13 +1827,13 @@ class GeoExplorer:
                         continue
                     loaded_data_sorted[key] = df.with_columns(
                         _unique_id=_get_unique_id(self._max_unique_id_int)
-                    ).drop("id", errors="ignore")
+                    ).drop("id", strict=False)
             else:
                 x = _standardize_path(x)
                 df = self._loaded_data[x]
                 loaded_data_sorted[x] = df.with_columns(
                     _unique_id=_get_unique_id(self._max_unique_id_int)
-                ).drop("id", errors="ignore")
+                ).drop("id", strict=False)
                 self._max_unique_id_int += 1
 
         self._loaded_data = loaded_data_sorted
@@ -2157,7 +2199,7 @@ class GeoExplorer:
                 and query.startswith("pl.col")
             ):
                 query = f"{old_query}, {query}"
-            self._queries[path] = query
+            self.set_query(path, query)
             return query
 
         @callback(
@@ -2831,7 +2873,7 @@ class GeoExplorer:
                 self.color_dict = {}
             elif not column and triggered is None:
                 column = self.column
-            elif self._concatted_data is None:
+            if self._concatted_data is None:
                 return (
                     [],
                     None,
@@ -3070,7 +3112,7 @@ class GeoExplorer:
                     bins=bins,
                     opacity=opacity,
                     n_rows_per_path=n_rows_per_path,
-                    columns=self._columns,
+                    columns=self._columns(),
                     current_columns=current_columns,
                 )
                 results = [
@@ -3104,15 +3146,21 @@ class GeoExplorer:
                     ):
                         _read_files(self, [img_path], mask=bbox)
                     img_bbox = self._bbox_series.loc[img_path]
-                    clipped_bounds = img_bbox.intersection(bbox)
-                    if clipped_bounds.is_empty:
+                    clipped_bbox = img_bbox.intersection(bbox)
+                    if clipped_bbox.is_empty:
                         continue
                     try:
-                        ds = self._open_img_path_as_xarray(
-                            img_path, selected_path, clipped_bounds
+                        self._nc[selected_path].validate_code_block()
+                        ds = self._nc[selected_path].filter_ds(
+                            ds=self._loaded_data[img_path],
+                            bounds=clipped_bbox.bounds,
+                            path=img_path,
                         )
                     except Exception as e:
                         traceback.print_exc()
+                        print(
+                            "\nNote: the above was a print of the error traceback from invalid query of dataset"
+                        )
                         alerts.append(
                             dbc.Alert(
                                 f"{type(e).__name__}: {e}. (Traceback printed in terminal)",
@@ -3128,10 +3176,11 @@ class GeoExplorer:
                     if np.isnan(arr).any() and not np.all(np.isnan(arr)):
                         arr[np.isnan(arr)] = np.min(arr[~np.isnan(arr)])
 
-                    images[img_path] = (arr, clipped_bounds)
+                    images[img_path] = (arr, clipped_bbox)
 
             if images:
                 # make sure all single-band images are normalized by same extremities
+                debug_print(images)
                 vmin = np.min([np.min(x[0]) for x in images.values()])
                 vmax = np.min([np.max(x[0]) for x in images.values()])
 
@@ -3861,7 +3910,6 @@ class GeoExplorer:
                 deleted_files.add(path)
                 break
 
-        assert len(deleted_files) == 1, deleted_files
         deleted_files2 = set()
         for i, path2 in enumerate(list(self._loaded_data)):
             parts = Path(path2).parts
@@ -4000,7 +4048,7 @@ class GeoExplorer:
                 if query == self._queries.get(path):
                     continue
                 self._check_for_circular_queries(query, path)
-                self._queries[path] = query
+                self.set_query(path, query)
             except RecursionError as e:
                 out_alerts.append(
                     dbc.Alert(
@@ -4016,6 +4064,11 @@ class GeoExplorer:
             return out_alerts
         return None
 
+    def set_query(self, key: str, value: str | None) -> None:
+        self._queries[key] = value
+        if key in self._nc:
+            self._nc[key].code_block = value
+
     @time_method_call(_PROFILE_DICT)
     def _get_unique_stem(self, path) -> str:
         name = _get_stem(path)
@@ -4023,19 +4076,6 @@ class GeoExplorer:
             name = _get_stem_from_parent(path)
         return name
 
-    def _open_img_path_as_xarray(self, img_path, selected_path, clipped_bounds):
-        if is_netcdf(img_path):
-            return self._nc[selected_path].filter_ds(
-                ds=self._loaded_data[img_path],
-                bounds=clipped_bounds.bounds,
-                code_block=self._queries.get(selected_path),
-            )
-        else:
-            return rasterio_to_xarray(
-                img_path, clipped_bounds, code_block=self._queries.get(selected_path)
-            )
-
-    @property
     def _columns(self) -> dict[str, set[str]]:
         return {path: set(dtypes) for path, dtypes in self._dtypes.items()} | {
             path: {"value"} for path in self._nc
@@ -4759,74 +4799,6 @@ def get_numeric_colors(values_no_nans_unique, values_no_nans, cmap, k):
     return color_dict, bins
 
 
-def rasterio_to_numpy(
-    img_path, bbox, return_attrs: list[str] | None = None
-) -> np.ndarray | tuple[Any]:
-    with rasterio.open(img_path) as src:
-        bounds_in_img_crs = GeoSeries([bbox], crs=4326).to_crs(src.crs).total_bounds
-        window = rasterio.windows.from_bounds(
-            *bounds_in_img_crs, transform=src.transform
-        )
-        arr = src.read(window=window, boundless=False, masked=False)
-        if not return_attrs:
-            return arr
-        return (arr, *[getattr(src, attr) for attr in return_attrs])
-
-
-def rasterio_to_xarray(img_path, bbox, code_block):
-    import xarray as xr
-    from rioxarray.rioxarray import _generate_spatial_coords
-
-    arr, crs, descriptions = rasterio_to_numpy(
-        img_path, bbox, return_attrs=["crs", "descriptions"]
-    )
-    bounds_in_img_crs = GeoSeries([bbox], crs=4326).to_crs(crs).total_bounds
-
-    if not all(arr.shape):
-        return xr.DataArray(
-            arr,
-            dims=["y", "x"],
-            attrs={"crs": crs},
-        )
-
-    if len(arr.shape) == 2:
-        height, width = arr.shape
-    elif len(arr.shape) == 3 and arr.shape[0] == 1:
-        arr = arr[0]
-        height, width = arr.shape
-    elif len(arr.shape) == 3:
-        height, width = arr.shape[1:]
-    else:
-        raise ValueError(arr.shape)
-
-    transform = rasterio.transform.from_bounds(*bounds_in_img_crs, width, height)
-    coords = _generate_spatial_coords(transform, width, height)
-
-    if len(arr.shape) == 2:
-        ds = xr.DataArray(
-            arr,
-            coords=coords,
-            dims=["y", "x"],
-            attrs={"crs": crs},
-        )
-    else:
-        if len(descriptions) != arr.shape[0]:
-            descriptions = range(arr.shape[0])
-        ds = xr.Dataset(
-            {
-                desc: xr.DataArray(
-                    arr[i],
-                    coords=coords,
-                    dims=["y", "x"],
-                    attrs={"crs": crs},
-                    name=desc,
-                )
-                for i, desc in enumerate(descriptions)
-            }
-        )
-    return _run_code_block(ds, code_block)
-
-
 def as_sized_array(arr: np.ndarray) -> np.ndarray:
     try:
         len(arr)

geo_explorer/img.py

@@ -0,0 +1,298 @@
+import abc
+from typing import Any
+from typing import ClassVar
+
+import numpy as np
+import pandas as pd
+import pyproj
+import rasterio
+import shapely
+from geopandas import GeoDataFrame
+from geopandas import GeoSeries
+from shapely.geometry import Polygon
+
+try:
+    from xarray import DataArray
+    from xarray import Dataset
+except ImportError:
+
+    class DataArray:
+        """Placeholder."""
+
+    class Dataset:
+        """Placeholder."""
+
+
+from .utils import _PROFILE_DICT
+from .utils import get_xarray_bounds
+from .utils import time_method_call
+
+
+class AbstractImageConfig(abc.ABC):
+    rgb_bands: ClassVar[list[str] | None] = None
+
+    def __init__(self, code_block: str | None = None) -> None:
+        self.code_block = code_block
+        self.validate_code_block()
+
+    @abc.abstractmethod
+    def get_crs(self, ds: Dataset, path: str) -> pyproj.CRS:
+        pass
+
+    @abc.abstractmethod
+    def get_bounds(self, ds: Dataset, path: str) -> tuple[float, float, float, float]:
+        pass
+
+    @time_method_call(_PROFILE_DICT)
+    def filter_ds(
+        self,
+        ds: Dataset,
+        path: str,
+        bounds: tuple[float, float, float, float],
+    ) -> GeoDataFrame | None:
+        crs = self.get_crs(ds, None)
+        ds_bounds = get_xarray_bounds(ds)
+
+        bbox_correct_crs = (
+            GeoSeries([shapely.box(*bounds)], crs=4326).to_crs(crs).union_all()
+        )
+        clipped_bbox = bbox_correct_crs.intersection(shapely.box(*ds_bounds))
+        minx, miny, maxx, maxy = clipped_bbox.bounds
+
+        x = "x" if "x" in list(ds.coords) else "X"
+        y = "y" if "y" in list(ds.coords) else "Y"
+        ds = ds.sel(**{x: slice(minx, maxx), y: slice(maxy, miny)})
+        return self._run_code_block(ds)
+
+    @time_method_call(_PROFILE_DICT)
+    def to_numpy(self, xarr: Dataset | DataArray) -> GeoDataFrame | None:
+        if isinstance(xarr, Dataset) and len(xarr.data_vars) == 1:
+            xarr = xarr[next(iter(xarr.data_vars))]
+        elif isinstance(xarr, Dataset):
+            try:
+                xarr = xarr[self.rgb_bands]
+            except Exception:
+                pass
+
+        if "time" in set(xarr.dims) and (
+            not hasattr(xarr["time"].values, "__len__") or len(xarr["time"].values) > 1
+        ):
+            print("Note: selecting first DataArray in time dimension.")
+            xarr = xarr.isel(time=0)
+
+        if isinstance(xarr, Dataset):
+            try:
+                return np.array([xarr[band].values for band in self.rgb_bands])
+            except Exception:
+                pass
+            if len(xarr.data_vars) == 3:
+                try:
+                    return np.array([xarr[band].values for band in xarr.data_vars])
+                except Exception:
+                    pass
+            arrs = []
+            for var in xarr.data_vars:
+                arr = xarr[var].values
+                if len(arr.shape) == 2:
+                    arrs.append(arr)
+                if len(arr.shape) == 3:
+                    arrs.append(arr[0])
+            return arrs[np.argmax([arr.shape for arr in arrs])]
+
+        if isinstance(xarr, np.ndarray):
+            return xarr
+
+        return xarr.values
+
+    def validate_code_block(self) -> None:
+        if not self.code_block:
+            return
+        try:
+            assert "ds" in self.code_block
+            eval(self.code_block)
+        except NameError:
+            return
+        except SyntaxError:
+            pass
+        if "xarr=" not in self.code_block.replace(" ", "") or not any(
+            txt in self.code_block.replace(" ", "") for txt in ("=ds", "(ds")
+        ):
+            raise ValueError(
+                "'code_block' must be a piece of code that takes the xarray object 'ds' and defines the object 'xarr'. "
+                f"Got '{self.code_block}'"
+            )
+
+    def _run_code_block(self, ds: DataArray | Dataset) -> Dataset | DataArray:
+        if not self.code_block:
+            return ds
+
+        try:
+            xarr = eval(self.code_block)
+            if callable(xarr):
+                xarr = xarr(ds)
+        except SyntaxError:
+            loc = {}
+            exec(self.code_block, globals() | {"ds": ds}, loc)
+            xarr = loc["xarr"]
+
+        if isinstance(xarr, np.ndarray) and isinstance(ds, DataArray):
+            ds.values = xarr
+            return ds
+        elif isinstance(xarr, np.ndarray) and isinstance(ds, Dataset):
+            raise ValueError(
+                "Cannot return np.ndarray from 'code_block' if ds is xarray.Dataset."
+            )
+        return xarr
+
+    def __str__(self) -> str:
+        code_block = f"'{self.code_block}'" if self.code_block else None
+        return f"{self.__class__.__name__}({code_block})"
+
+    def __repr__(self) -> str:
+        return str(self)
+
+
+class NetCDFConfig(AbstractImageConfig):
+    """Sets the configuration for reading NetCDF files and getting crs and bounds.
+
+    Args:
+        code_block: String of Python code that takes an xarray.Dataset (ds) and returns an xarray.DataArray (xarr).
+            Note that the input Dataset must be references as 'ds' and the output must be assigned to 'xarr'.
+    """
+
+    rgb_bands: ClassVar[list[str]] = ["B4", "B3", "B2"]
+
+    def get_bounds(self, ds, path) -> tuple[float, float, float, float]:
+        return get_xarray_bounds(ds)
+
+    def get_crs(self, ds: Dataset, path: str) -> pyproj.CRS:
+        attrs = [
+            x
+            for x in set(ds.attrs).union(
+                set(ds.data_vars if isinstance(ds, Dataset) else set())
+            )
+            if "projection" in x.lower() or "crs" in x.lower() or "utm" in x.lower()
+        ]
+        if not attrs:
+            raise ValueError(f"Could not find CRS attribute/data_var in dataset: {ds}")
+
+        def getattr_xarray(ds, attr):
+            x = ds.attrs.get(attr, ds.get(attr))
+            if isinstance(x, DataArray):
+                try:
+                    return pyproj.CRS(str(x.attrs["proj4"]))
+                except Exception:
+                    return pyproj.CRS(str(x.values))
+            elif x is not None:
+                return pyproj.CRS(x)
+            raise ValueError(f"Could not find CRS attribute/data_var in dataset: {ds}")
+
+        for i, attr in enumerate(attrs):
+            try:
+                return getattr_xarray(ds, attr)
+            except Exception as e:
+                if i == len(attrs) - 1:
+                    attrs_dict = {attr: ds.attrs.get(attr, ds[attr]) for attr in attrs}
+                    raise ValueError(
+                        f"No valid CRS attribute found among {attrs_dict}"
+                    ) from e
+
+
+class NBSNetCDFConfig(NetCDFConfig):
+    def get_crs(self, ds: Dataset, path: str) -> pyproj.CRS:
+        return pyproj.CRS(ds.UTM_projection.epsg_code)
+
+
+class Sentinel2NBSNetCDFConfig(NBSNetCDFConfig):
+    rgb_bands: ClassVar[list[str]] = ["B4", "B3", "B2"]
+
+
+class GeoTIFFConfig(AbstractImageConfig):
+    def get_crs(self, ds, path):
+        with rasterio.open(path) as src:
+            return src.crs
+
+    def get_bounds(self, ds, path) -> tuple[float, float, float, float]:
+        with rasterio.open(path) as src:
+            return tuple(
+                GeoSeries([shapely.box(*src.bounds)], crs=src.crs)
+                .to_crs(4326)
+                .total_bounds
+            )
+
+    def filter_ds(
+        self,
+        ds: None,
+        path: str,
+        bounds: tuple[float, float, float, float],
+    ) -> DataArray | Dataset:
+        from rioxarray.rioxarray import _generate_spatial_coords
+
+        bbox = shapely.box(*bounds)
+        arr, crs, descriptions = rasterio_to_numpy(
+            path, bbox, return_attrs=["crs", "descriptions"]
+        )
+        bounds_in_img_crs = GeoSeries([bbox], crs=4326).to_crs(crs).total_bounds
+
+        if not all(arr.shape):
+            return DataArray(
+                arr,
+                dims=["y", "x"],
+                attrs={"crs": crs},
+            )
+
+        if len(arr.shape) == 2:
+            height, width = arr.shape
+        elif len(arr.shape) == 3 and arr.shape[0] == 1:
+            arr = arr[0]
+            height, width = arr.shape
+        elif len(arr.shape) == 3:
+            height, width = arr.shape[1:]
+        else:
+            raise ValueError(arr.shape)
+
+        transform = rasterio.transform.from_bounds(*bounds_in_img_crs, width, height)
+        coords = _generate_spatial_coords(transform, width, height)
+
+        if len(arr.shape) == 2:
+            ds = DataArray(
+                arr,
+                coords=coords,
+                dims=["y", "x"],
+                attrs={"crs": crs},
+            )
+        else:
+            if len(descriptions) != arr.shape[0]:
+                descriptions = range(arr.shape[0])
+            ds = Dataset(
+                {
+                    desc: DataArray(
+                        arr[i],
+                        coords=coords,
+                        dims=["y", "x"],
+                        attrs={"crs": crs},
+                        name=desc,
+                    )
+                    for i, desc in enumerate(descriptions)
+                }
+            )
+        return self._run_code_block(ds)
+
+
+def rasterio_to_numpy(
+    img_path: str, bbox: Polygon, return_attrs: list[str] | None = None
+) -> np.ndarray | tuple[Any]:
+    with rasterio.open(img_path) as src:
+        bounds_in_img_crs = GeoSeries([bbox], crs=4326).to_crs(src.crs).total_bounds
+        window = rasterio.windows.from_bounds(
+            *bounds_in_img_crs, transform=src.transform
+        )
+        arr = src.read(window=window, boundless=False, masked=False)
+        if not return_attrs:
+            return arr
+        return (arr, *[getattr(src, attr) for attr in return_attrs])
+
+
+def _pd():
+    """Function that makes sure 'pd' is not removed by 'ruff' fixes. Because pd is useful in code_block."""
+    pd

geo_explorer/utils.py

@@ -87,12 +87,45 @@ if not DEBUG:
         return decorator
 
 
+def get_xarray_resolution(ds) -> int:
+    minx, miny, maxx, maxy = _get_raw_xarray_bounds(ds)
+    diffx = maxx - minx
+    diffy = maxy - miny
+    x = "x" if "x" in list(ds.coords) else "X"
+    y = "y" if "y" in list(ds.coords) else "Y"
+    try:
+        resx = diffx / (ds.sizes[x] - 1)
+        resy = diffy / (ds.sizes[y] - 1)
+    except ZeroDivisionError:
+        raise ValueError(
+            f"Cannot calculate resolution for Dataset with {diffx=}, {diffy=}, {ds.sizes['x']=}, {ds.sizes['y']=}"
+        )
+    if resx != resy:
+        raise ValueError(
+            f"x and y resolution differ: resx={resx}, resy={resy} for Dataset: {ds}"
+        )
+    return resx
+
+
 def get_xarray_bounds(ds) -> tuple[float, float, float, float]:
+    res = get_xarray_resolution(ds)
+    minx, miny, maxx, maxy = _get_raw_xarray_bounds(ds)
+    return (
+        minx - res / 2,
+        miny - res / 2,
+        maxx + res / 2,
+        maxy + res / 2,
+    )
+
+
+def _get_raw_xarray_bounds(ds) -> tuple[float, float, float, float]:
+    x = "x" if "x" in list(ds.coords) else "X"
+    y = "y" if "y" in list(ds.coords) else "Y"
     return (
-        float(ds["x"].min().values),
-        float(ds["y"].min().values),
-        float(ds["x"].max().values),
-        float(ds["y"].max().values),
+        float(ds[x].min().values),
+        float(ds[y].min().values),
+        float(ds[x].max().values),
+        float(ds[y].max().values),
     )
 
 

{geo_explorer-0.9.10.dist-info → geo_explorer-0.9.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: geo-explorer
-Version: 0.9.10
+Version: 0.9.12
 Summary: Explore geodata interactively.
 License: MIT
 Author: Morten Letnes

geo_explorer-0.9.12.dist-info/RECORD

@@ -0,0 +1,14 @@
+geo_explorer/__init__.py,sha256=LAnELq1tD-bpPHVM_bPa8DqAD4kY6dE9ZalZSp3R0Ug,108
+geo_explorer/assets/chroma.min.js,sha256=cNBWTSvPBSwnJYaqi4SM3IFXRjpWCLsa4VVfJ8yYWNw,46173
+geo_explorer/assets/on_each_feature.js,sha256=pjpr3MGlaTZf8pa191hRB5Iui8un7jBG5mFw_GNGqxE,2630
+geo_explorer/assets/stylesheet.css,sha256=1Dow3oh532wDZMEFr-VlkCdfekuf_20K97AVKatJun0,1957
+geo_explorer/file_browser.py,sha256=Ap1cx8WOX-A1oZgo8aQ2DKPIIGxJd1CscLFe9GHKs8M,23896
+geo_explorer/fs.py,sha256=N_WtpAjohO6OKAzoWzNxko0Rv_Y4oH0I8ackcoPu4f8,1693
+geo_explorer/geo_explorer.py,sha256=AyDk3jXwGwz6HyE-6phw2fidXDqD3cIVpOFG6zLKJ94,191686
+geo_explorer/img.py,sha256=1PIJgFe597wgbYYPNQ-RWaaYbbcg8qz3LJ9FEg7lw0s,9808
+geo_explorer/utils.py,sha256=n7ttTCJjShabBGnKEfmflw15ccMJqTDVcOiFnJ_SokE,4270
+geo_explorer-0.9.12.dist-info/LICENSE,sha256=lL2h0dNKGTKAE0CjTy62SDbRennVD1xPgM5LzGqhKeo,1074
+geo_explorer-0.9.12.dist-info/LICENSE.md,sha256=hxspefYgWP3U6OZFhCifqWMI5ksnKzgFxNKgQnG7Ozc,1074
+geo_explorer-0.9.12.dist-info/METADATA,sha256=X5k_Gnb31D_cm-RYAd6b8vDulEaNTxZITTwaDRcps8U,8488
+geo_explorer-0.9.12.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+geo_explorer-0.9.12.dist-info/RECORD,,

geo_explorer/nc.py

@@ -1,199 +0,0 @@
-import abc
-from typing import ClassVar
-
-import numpy as np
-import pandas as pd
-import pyproj
-import rasterio
-import shapely
-from geopandas import GeoDataFrame
-from geopandas import GeoSeries
-from shapely.geometry import Polygon
-
-try:
-    from xarray import DataArray
-    from xarray import Dataset
-except ImportError:
-
-    class DataArray:
-        """Placeholder."""
-
-    class Dataset:
-        """Placeholder."""
-
-
-from .utils import _PROFILE_DICT
-from .utils import get_xarray_bounds
-from .utils import time_method_call
-
-
-class AbstractImageConfig(abc.ABC):
-    rgb_bands: ClassVar[list[str] | None] = None
-    reducer: ClassVar[str | None] = None
-
-    def __init__(self, code_block: str | None = None) -> None:
-        self._code_block = code_block
-        self.code_block = code_block  # trigger setter
-
-    @property
-    def code_block(self) -> str | None:
-        return self._code_block
-
-    @code_block.setter
-    def code_block(self, value: str | None):
-        if value and (
-            "xarr=" not in value.replace(" ", "")
-            or not any(txt in value.replace(" ", "") for txt in ("=ds", "(ds"))
-        ):
-            raise ValueError(
-                "'code_block' must be a piece of code that takes the xarray object 'ds' and defines the object 'xarr'. "
-                f"Got '{value}'"
-            )
-        self._code_block = value
-
-    @abc.abstractmethod
-    def get_crs(self, ds: Dataset, path: str) -> pyproj.CRS:
-        pass
-
-    @abc.abstractmethod
-    def get_bounds(self, ds: Dataset, path: str) -> tuple[float, float, float, float]:
-        pass
-
-    @time_method_call(_PROFILE_DICT)
-    def filter_ds(
-        self,
-        ds: Dataset,
-        bounds: tuple[float, float, float, float],
-        code_block: str | None,
-    ) -> GeoDataFrame | None:
-        crs = self.get_crs(ds, None)
-        ds_bounds = get_xarray_bounds(ds)
-
-        bbox_correct_crs = (
-            GeoSeries([shapely.box(*bounds)], crs=4326).to_crs(crs).union_all()
-        )
-        clipped_bbox = bbox_correct_crs.intersection(shapely.box(*ds_bounds))
-        minx, miny, maxx, maxy = clipped_bbox.bounds
-
-        ds = ds.sel(
-            x=slice(minx, maxx),
-            y=slice(maxy, miny),
-        )
-
-        return _run_code_block(ds, code_block)
-
-    @time_method_call(_PROFILE_DICT)
-    def to_numpy(self, xarr: Dataset | DataArray) -> GeoDataFrame | None:
-        if isinstance(xarr, Dataset) and len(xarr.data_vars) == 1:
-            xarr = xarr[next(iter(xarr.data_vars))]
-        elif isinstance(xarr, Dataset):
-            try:
-                xarr = xarr[self.rgb_bands]
-            except Exception:
-                pass
-
-        if "time" in set(xarr.dims) and (
-            not hasattr(xarr["time"].values, "__len__") or len(xarr["time"].values) > 1
-        ):
-            if self.reducer is None:
-                xarr = xarr.isel(time=0)
-            else:
-                xarr = getattr(xarr, self.reducer)(dim="time")
-
-        if isinstance(xarr, Dataset) and self.rgb_bands:
-            return np.array([xarr[band].values for band in self.rgb_bands])
-        elif isinstance(xarr, Dataset):
-            return np.array([xarr[var].values for var in xarr.data_vars])
-
-        if isinstance(xarr, np.ndarray):
-            return xarr
-
-        return xarr.values
-
-    def __str__(self) -> str:
-        code_block = f"'{self.code_block}'" if self.code_block else None
-        return f"{self.__class__.__name__}({code_block})"
-
-    def __repr__(self) -> str:
-        return str(self)
-
-
-def _run_code_block(
-    ds: DataArray | Dataset, code_block: str | None
-) -> Dataset | DataArray:
-    if not code_block:
-        return ds
-
-    try:
-        xarr = eval(code_block)
-        if callable(xarr):
-            xarr = xarr(ds)
-    except SyntaxError:
-        loc = {}
-        exec(code_block, globals() | {"ds": ds}, loc)
-        xarr = loc["xarr"]
-
-    if isinstance(xarr, np.ndarray) and isinstance(ds, DataArray):
-        ds.values = xarr
-        return ds
-    elif isinstance(xarr, np.ndarray) and isinstance(ds, Dataset):
-        raise ValueError(
-            "Cannot return np.ndarray from 'code_block' if ds is xarray.Dataset."
-        )
-    return xarr
-
-
-class GeoTIFFConfig(AbstractImageConfig):
-    def get_crs(self, ds, path):
-        with rasterio.open(path) as src:
-            return src.crs
-
-    def get_bounds(self, ds, path) -> tuple[float, float, float, float]:
-        with rasterio.open(path) as src:
-            return tuple(
-                GeoSeries([shapely.box(*src.bounds)], crs=src.crs)
-                .to_crs(4326)
-                .total_bounds
-            )
-
-
-class NetCDFConfig(AbstractImageConfig):
-    """Sets the configuration for reading NetCDF files and getting crs and bounds.
-
-    Args:
-        code_block: String of Python code that takes an xarray.Dataset (ds) and returns an xarray.DataArray (xarr).
-            Note that the input Dataset must be references as 'ds' and the output must be assigned to 'xarr'.
-    """
-
-    rgb_bands: ClassVar[list[str]] = ["B4", "B3", "B2"]
-
-    def get_bounds(self, ds, path) -> tuple[float, float, float, float]:
-        return get_xarray_bounds(ds)
-
-    def get_crs(self, ds: Dataset, path: str) -> pyproj.CRS:
-        attrs = [x for x in ds.attrs if "projection" in x.lower() or "crs" in x.lower()]
-        if not attrs:
-            raise ValueError(f"Could not find CRS attribute in dataset: {ds}")
-        for i, attr in enumerate(attrs):
-            try:
-                return pyproj.CRS(ds.attrs[attr])
-            except Exception as e:
-                if i == len(attrs) - 1:
-                    attrs_dict = {attr: ds.attrs[attr] for attr in attrs}
-                    raise ValueError(
-                        f"No valid CRS attribute found among {attrs_dict}"
-                    ) from e
-
-
-class NBSNetCDFConfig(NetCDFConfig):
-    def get_crs(self, ds: Dataset, path: str) -> pyproj.CRS:
-        return pyproj.CRS(ds.UTM_projection.epsg_code)
-
-
-class Sentinel2NBSNetCDFConfig(NBSNetCDFConfig):
-    rgb_bands: ClassVar[list[str]] = ["B4", "B3", "B2"]
-
-
-def _pd():
-    """Function that makes sure 'pd' is not removed by 'ruff' fixes. Because pd is useful in code_block."""
-    pd

geo_explorer-0.9.10.dist-info/RECORD

@@ -1,14 +0,0 @@
-geo_explorer/__init__.py,sha256=LAnELq1tD-bpPHVM_bPa8DqAD4kY6dE9ZalZSp3R0Ug,108
-geo_explorer/assets/chroma.min.js,sha256=cNBWTSvPBSwnJYaqi4SM3IFXRjpWCLsa4VVfJ8yYWNw,46173
-geo_explorer/assets/on_each_feature.js,sha256=pjpr3MGlaTZf8pa191hRB5Iui8un7jBG5mFw_GNGqxE,2630
-geo_explorer/assets/stylesheet.css,sha256=1Dow3oh532wDZMEFr-VlkCdfekuf_20K97AVKatJun0,1957
-geo_explorer/file_browser.py,sha256=NmMAKHznJ1TOJ8TH-axwfvKaZ7r0gH9QSCmVCXHnfZw,23270
-geo_explorer/fs.py,sha256=N_WtpAjohO6OKAzoWzNxko0Rv_Y4oH0I8ackcoPu4f8,1693
-geo_explorer/geo_explorer.py,sha256=bELLu8lp8HCBr_HLmCWGqHNvoHzfEQy9VSr56nVqqrE,192235
-geo_explorer/nc.py,sha256=nmCa_FJ0icBeL9R9aWt2Fhvng7Drfbsf1whynC8Wm98,6188
-geo_explorer/utils.py,sha256=NIu5l-jCvOOTJtGRNRZlCUDGOtk5Q0zf7F1kLg8idMs,3238
-geo_explorer-0.9.10.dist-info/LICENSE,sha256=lL2h0dNKGTKAE0CjTy62SDbRennVD1xPgM5LzGqhKeo,1074
-geo_explorer-0.9.10.dist-info/LICENSE.md,sha256=hxspefYgWP3U6OZFhCifqWMI5ksnKzgFxNKgQnG7Ozc,1074
-geo_explorer-0.9.10.dist-info/METADATA,sha256=2jgEwFtoZzvbtQClP9bLOcXDRMk7WHKxZXdvKt7ugGA,8488
-geo_explorer-0.9.10.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-geo_explorer-0.9.10.dist-info/RECORD,,