pygeodata 0.0.0__tar.gz

pygeodata-0.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Jasper Roebroek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pygeodata-0.0.0/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.4
+Name: pygeodata
+Version: 0.0.0
+Summary: Lightweight framework for geospatial data access and processing
+Author-email: Jasper Roebroek <roebroek.jasper@gmail.com>
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: rioxarray
+Requires-Dist: geopandas
+Requires-Dist: dask
+Requires-Dist: zarr
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Dynamic: license-file
+
+# pygeodata
+
+**Lightweight framework for geospatial data access and processing**
+
+`pygeodata` provides simple, composable tools for working with geospatial raster and vector data. It aims to make spatial data handling consistent, efficient, and compatible with large-scale and cloud-based workflows.
+
+## Features
+
+- Unified access to raster and vector datasets  
+- CRS-aware reprojection and alignment utilities
+- Clean, composable API designed for scientific workflows  
+
+## Installation
+
+```bash
+pip install pygeodata
+```

pygeodata-0.0.0/README.md

@@ -0,0 +1,17 @@
+# pygeodata
+
+**Lightweight framework for geospatial data access and processing**
+
+`pygeodata` provides simple, composable tools for working with geospatial raster and vector data. It aims to make spatial data handling consistent, efficient, and compatible with large-scale and cloud-based workflows.
+
+## Features
+
+- Unified access to raster and vector datasets  
+- CRS-aware reprojection and alignment utilities
+- Clean, composable API designed for scientific workflows  
+
+## Installation
+
+```bash
+pip install pygeodata
+```

pygeodata-0.0.0/pygeodata/__init__.py

@@ -0,0 +1,12 @@
+from pygeodata.base import load, process
+from pygeodata.config import set_config
+from pygeodata.loader import DataLoader
+
+__version__ = '0.0.2'
+
+__all__ = [
+    'DataLoader',
+    'load',
+    'process',
+    'set_config',
+]

pygeodata-0.0.0/pygeodata/base.py

@@ -0,0 +1,21 @@
+from typing import Any
+
+from pygeodata.config import get_config
+from pygeodata.loader import DataLoader
+from pygeodata.types import SpatialSpec
+
+
+def load(loader: DataLoader, spec: SpatialSpec | None = None) -> Any:
+    spec = spec or get_config().spec
+    if spec is None:
+        raise ValueError('No spatial specification (spec) provided or present in config')
+    return loader(spec)
+
+
+def process(loader: DataLoader, spec: SpatialSpec | None = None) -> None:
+    spec = spec or get_config().spec
+    if spec is None:
+        raise ValueError('No spatial specification (spec) provided or present in config')
+    if loader.is_processed(spec):
+        return
+    loader.process(spec)

pygeodata-0.0.0/pygeodata/config.py

@@ -0,0 +1,40 @@
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from typing import Any
+
+from pygeodata.options import RasterCreationOptions
+from pygeodata.types import SpatialSpec
+
+
+@dataclass
+class Config:
+    path_data_processed: Path = Path('data_processed')
+    num_threads: int = 1
+    warp_mem_limit: int = 0  # GDAL default, indicates 64 MB
+    spec: SpatialSpec | None = None
+    raster_creation_options: RasterCreationOptions = field(default_factory=RasterCreationOptions)
+
+    def update(self, **kwargs):
+        for key, value in kwargs.items():
+            if not hasattr(self, key):
+                raise ValueError(f'Invalid config key: {key}')
+            setattr(self, key, value)
+
+
+CONFIG = Config()
+
+
+def get_config() -> Config:
+    return CONFIG
+
+
+@contextmanager
+def set_config(**overrides: Any) -> Iterator[Config]:
+    old_values = {k.name: getattr(CONFIG, k.name) for k in fields(CONFIG)}
+    CONFIG.update(**overrides)
+    try:
+        yield CONFIG
+    finally:
+        CONFIG.update(**old_values)

pygeodata-0.0.0/pygeodata/drivers/__init__.py

@@ -0,0 +1,3 @@
+from pygeodata.drivers.rioxarray import RioXArrayDriver
+
+__all__ = ['RioXArrayDriver']

pygeodata-0.0.0/pygeodata/drivers/rioxarray.py

@@ -0,0 +1,92 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import numpy as np
+import rasterio as rio
+import rioxarray as rxr
+import xarray as xr
+from rioxarray.exceptions import TooManyDimensions
+
+
+@dataclass
+class RioXArrayDriver:
+    """Load a raster file using rioxarray.
+
+    Parameters
+    ----------
+    parse_coordinates : bool, optional
+        Whether to parse coordinates
+    mask_and_scale : bool, optional
+        Whether to mask and scale
+    decode_times : bool, optional
+        Whether to decode times, by default False
+    cache : bool, optional
+        Whether to cache, by default False
+    flatten : bool, optional
+        By default 2D rasters will be returned as 3D, with a band dimension of size 1. If True, which is the
+        default, this dimension is removed.
+    boolean : bool, optional
+        Whether to convert the data to boolean, by default False
+    open_kw : dict, optional
+        Additional keyword arguments to pass to rioxarray.open_rasterio
+    """
+
+    parse_coordinates: bool = True
+    mask_and_scale: bool = True
+    decode_times: bool = False
+    cache: bool = False
+    flatten: bool = True
+    boolean: bool = False
+    open_kw: dict = field(default_factory=dict)
+
+    @staticmethod
+    def _subdatasets(path: Path) -> list[str]:
+        with rio.open(path) as fp:
+            return fp.subdatasets
+
+    def _has_subdatasets(self, path: Path) -> bool:
+        return len(self._subdatasets(path)) > 0
+
+    @staticmethod
+    def _infer_mask_and_scale(path: Path) -> bool:
+        with rio.open(path) as fp:
+            return np.issubdtype(fp.dtypes[0], np.floating)
+
+    @staticmethod
+    def _assert_exists(path: Path) -> None:
+        with rio.open(path):
+            pass
+
+    def __call__(self, path: str | Path) -> xr.DataArray:
+        path = Path(path)
+
+        self._assert_exists(path)
+
+        if self._has_subdatasets(path):
+            subs_str = '\n'.join(self._subdatasets(path))
+            raise TooManyDimensions(f'Multiple variables found in {path}. Use one of the subdatasets:\n{subs_str}')
+
+        try:
+            da = rxr.open_rasterio(
+                path,
+                parse_coordinates=self.parse_coordinates,
+                decode_times=self.decode_times,
+                cache=self.cache,
+                mask_and_scale=self.mask_and_scale,
+                **self.open_kw,
+            )
+        except Exception as e:
+            raise type(e)(f'Failed to load {path}: {e}') from e
+
+        if isinstance(da, xr.Dataset):
+            raise TooManyDimensions(f'Dataset found in {path}. Use one of the subdatasets.')
+
+        if self.flatten and 'band' in da.dims and da.sizes['band'] == 1:
+            da = da.isel(band=0).drop_vars('band')
+
+        if self.boolean:
+            da = da.astype(bool)
+
+        return da
+
+    default_ext = 'tif'

pygeodata-0.0.0/pygeodata/loader.py

@@ -0,0 +1,90 @@
+import re
+from pathlib import Path
+from typing import Any
+
+from pygeodata.config import get_config
+from pygeodata.paths import generate_path
+from pygeodata.types import Driver, Processor, SpatialSpec
+
+
+class DataLoader:
+    @property
+    def processor(self) -> Processor:
+        raise NotImplementedError(f'{self}: Either processor or driver must be implemented')
+
+    @property
+    def driver(self) -> Driver:
+        try:
+            processor = self.processor
+        except NotImplementedError:
+            raise NotImplementedError(f'{self}: Either processor or driver must be implemented')
+
+        if not hasattr(processor, 'default_driver'):
+            raise AttributeError(f'Processor {processor} lacks default_driver and no driver is set')
+        return getattr(processor, 'default_driver')
+
+    @property
+    def class_name(self) -> str:
+        return self.__class__.__name__.split('.')[-1].replace('Loader', '')
+
+    @property
+    def name(self) -> str:
+        # Handle acronym → word transitions (e.g. XMLHTTPRequest → XML_Http_Request)
+        s1 = re.sub('([A-Z]+)([A-Z][a-z])', r'\1_\2', self.class_name)
+        # Handle normal camelCase → camel_Case
+        s2 = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1)
+        return s2.lower()
+
+    @property
+    def ext(self) -> str:
+        try:
+            ext = getattr(self.processor, 'ext', None)
+        except NotImplementedError:
+            ext = None
+        if ext is None:
+            ext = self.driver.default_ext
+        return ext
+
+    def get_params(self) -> dict[str, Any]:
+        params = {}
+        for key in self.__dict__:
+            if key in ('name', 'class_name', 'processor', 'driver', 'process', 'load'):
+                continue
+            if key.startswith('_'):
+                continue
+            params.update({key: self.__dict__[key]})
+        return params
+
+    def __repr__(self) -> str:
+        params = self.get_params()
+        parts = [f'{k}={v!r}' for k, v in sorted(params.items())]
+        return f'{self.class_name}({", ".join(parts)})'
+
+    def get_processed_path(self, spec: SpatialSpec, ext: str | None = None) -> Path:
+        ext = ext or self.ext
+
+        path = generate_path(
+            spec=spec,
+            name=self.class_name,
+            filename=self.name,
+            base_dir=get_config().path_data_processed,
+            ext=ext,
+            **self.get_params(),
+        )
+        path.parent.mkdir(exist_ok=True, parents=True)
+        return path
+
+    def is_processed(self, spec: SpatialSpec) -> bool:
+        p = self.get_processed_path(spec)
+        return p.exists()
+
+    def process(self, spec: SpatialSpec) -> None:
+        self.processor(self.get_processed_path(spec), spec)
+
+    def load(self, spec: SpatialSpec) -> Any:
+        return self.driver(self.get_processed_path(spec))
+
+    def __call__(self, spec: SpatialSpec) -> Any:
+        if not self.is_processed(spec):
+            self.process(spec)
+        return self.load(spec)

pygeodata-0.0.0/pygeodata/options.py

@@ -0,0 +1,45 @@
+from dataclasses import asdict, dataclass
+from typing import Any
+
+
+@dataclass
+class RasterCreationOptions:
+    """Rasterio creation profile options.
+
+    Parameters
+    ----------
+    compress : str, optional
+        Compression algorithm ('lzw', 'deflate', 'zstd', 'lzma', 'jpeg', 'webp')
+    compress_level : int, optional
+        Compression level (1-9 for deflate, 1-12 for zstd)
+    tiled : bool, default=False
+        Whether to create a tiled raster (False for striped layout)
+    blockxsize : int, optional
+        Tile width (defaults to 256 if tiled=True, must be multiple of 16)
+    blockysize : int, optional
+        Tile height (defaults to 256 if tiled=True, must be multiple of 16)
+    interleave : str, optional
+        Band interleave ('pixel', 'band', 'line')
+    photometric : str, optional
+        Photometric interpretation ('minisblack', 'rgb', 'ycbcr')
+    predictor : int, optional
+        Predictor for compression (1=none, 2=horizontal, 3=floating point)
+    bigtiff : bool | str, optional
+        Create BigTIFF file ('yes', 'no', 'if_needed', 'if_safer')
+    sparse_ok : bool, optional
+        Allow sparse files
+    """
+
+    compress: str | None = None
+    compress_level: int | None = None
+    tiled: bool | None = None
+    blockxsize: int | None = None
+    blockysize: int | None = None
+    interleave: str | None = None
+    photometric: str | None = None
+    predictor: int | None = None
+    bigtiff: bool | str | None = None
+    sparse_ok: bool | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {k: v for k, v in asdict(self).items() if v is not None}

pygeodata-0.0.0/pygeodata/paths.py

@@ -0,0 +1,33 @@
+import re
+from pathlib import Path
+
+from pygeodata.types import SpatialSpec
+from pygeodata.utils import transform_to_str
+
+
+def generate_path(
+    spec: SpatialSpec,
+    base_dir: str | Path,
+    filename: str,
+    ext: str,
+    name: str | None = None,
+    **kwargs,
+) -> Path:
+    """Function that converts a path of the data to the processed data."""
+    base_dir = Path(base_dir)
+
+    p = []
+    if name is not None:
+        p.append(name)
+
+    for key in sorted(kwargs.keys()):
+        p.append(f'{key}={kwargs[key]}')
+
+    return Path(
+        base_dir,
+        re.sub(r'[^\w\-]', '_', spec.crs.to_string()),
+        transform_to_str(spec.transform),
+        f'{spec.shape[0]}-{spec.shape[1]}',
+        *p,
+        f'{filename}.{ext}',
+    )

pygeodata-0.0.0/pygeodata/processors/__init__.py

@@ -0,0 +1,4 @@
+from pygeodata.processors.rasterizer import Rasterizer
+from pygeodata.processors.reprojection import Reprojector
+
+__all__ = ['Rasterizer', 'Reprojector']

pygeodata-0.0.0/pygeodata/processors/rasterizer.py

@@ -0,0 +1,100 @@
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import geopandas as gpd
+import numpy as np
+import rasterio
+from rasterio.features import rasterize
+
+from pygeodata.config import get_config
+from pygeodata.drivers import RioXArrayDriver
+from pygeodata.options import RasterCreationOptions
+from pygeodata.types import SpatialSpec
+
+
+@dataclass
+class Rasterizer:
+    """
+    Rasterize a vector dataset to a single-band raster.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the vector dataset (e.g., shapefile, GeoPackage).
+    column : str
+        Name of the column to use as raster values (must be numeric).
+    load_df_func : Callable[[str | Path, SpatialSpec], gpd.GeoDataFrame], optional
+        Function to load the vector data. By default, reads with geopandas and reprojects to `spec.crs`.
+    all_touched : bool, default=True
+        Whether to burn all pixels touched by geometries.
+    dtype : np.dtype, optional
+        Data type for raster. Defaults to the dtype of `column`.
+    fill_value : float, optional
+        Nodata value for raster. Defaults to `_NODATA_DTYPE_MAP` based on dtype.
+    rasterize_kw : dict, optional
+        Additional keyword arguments passed to `rasterio.features.rasterize`.
+    raster_creation_options : RasterCreationOptions, optional
+        Optional raster creation profile (compression, tiling, etc.).
+    """
+
+    path: Path
+    column: str = 'index'
+    load_df_func: Callable[[str | Path, SpatialSpec], gpd.GeoDataFrame] = field(
+        default_factory=lambda: lambda path, spec: gpd.read_file(path).to_crs(spec.crs).reset_index(),
+    )
+    all_touched: bool = True
+    dtype: np.dtype | None = None
+    fill_value: float | None = None
+    rasterize_kw: dict[str, Any] = field(default_factory=dict)
+    raster_creation_options: RasterCreationOptions | None = None
+
+    def __call__(self, dst_path: str | Path, spec: SpatialSpec) -> None:
+        df = self.load_df_func(self.path, spec)
+
+        if df.crs != spec.crs:
+            raise ValueError(f'GeoDataFrame CRS ({df.crs}) does not match target spec CRS ({spec.crs}).')
+
+        if self.column not in df.columns:
+            raise ValueError(f"Column '{self.column}' not found in GeoDataFrame.")
+
+        dtype = self.dtype if self.dtype is not None else df[self.column].dtype
+
+        if not np.issubdtype(dtype, np.number):
+            raise TypeError(f"Column '{self.column}' must be numeric, got {dtype}.")
+
+        default_fill_value = np.nan if np.issubdtype(dtype, np.floating) else 0
+        fill_value = self.fill_value if self.fill_value is not None else default_fill_value
+
+        if fill_value in df[self.column]:
+            raise ValueError(f'Fill value {fill_value} is present in the data. Overwrite with a different value.')
+
+        raster = rasterize(
+            ((row.geometry, row[self.column]) for i, row in df.iterrows()),
+            out_shape=spec.shape,
+            transform=spec.transform,
+            fill=fill_value,
+            all_touched=self.all_touched,
+            dtype=dtype,
+            **self.rasterize_kw,
+        )
+
+        raster_creation_options = self.raster_creation_options or get_config().raster_creation_options
+
+        with rasterio.open(
+            dst_path,
+            'w',
+            driver='GTiff',
+            height=spec.shape[0],
+            width=spec.shape[1],
+            count=1,
+            dtype=dtype,
+            crs=spec.crs,
+            transform=spec.transform,
+            **raster_creation_options.to_dict(),
+        ) as dst:
+            dst.write(raster, 1)
+
+    default_driver = RioXArrayDriver()
+    ext = 'tif'

pygeodata-0.0.0/pygeodata/processors/reprojection.py

@@ -0,0 +1,168 @@
+import shutil
+import tempfile
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from numbers import Number
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import rasterio as rio
+import rasterio.warp
+from numpy.typing import DTypeLike
+from rasterio import CRS, RasterioIOError
+from rasterio.enums import Resampling
+
+from pygeodata.config import get_config
+from pygeodata.drivers import RioXArrayDriver
+from pygeodata.options import RasterCreationOptions
+from pygeodata.types import SpatialSpec
+
+
+@dataclass
+class Reprojector:
+    """Reprojects raster data to GeoTIFF format.
+
+    Parameters
+    ----------
+    src_path : str | Path
+        Path to source raster file
+    bands : int or tuple of int, optional
+        Band indices to reproject (1-indexed). If None, reprojects all bands
+    src_crs : CRS, optional
+        Override source CRS if not defined in file
+    resampling : Resampling, default=Resampling.nearest
+        Resampling method
+    dst_dtype : DTypeLike, optional
+        Output data type. If None, uses source dtype
+    dst_nodata : float, optional
+        Output nodata value. If None, uses source nodata
+    nbits : int, optional
+        Number of bits per pixel
+    warp_kw : dict, optional
+        Additional keyword arguments for rasterio.warp.reproject
+    scales : float or sequence of floats, optional
+        Scale factor for each band
+    offsets : float or sequence of floats, optional
+        Offset for each band
+    raster_creation_options : RasterCreationOptions, optional
+        GeoTIFF creation profile options. If None, uses defaults
+    """
+
+    src_path: str | Path
+    src_crs: CRS | None = None
+    bands: int | Sequence[int] | None = None
+    resampling: Resampling = Resampling.nearest
+    dst_dtype: DTypeLike | None = None
+    dst_nodata: float | None = None
+    nbits: int | None = None
+    warp_kw: dict[str, Any] = field(default_factory=dict)
+    warp_mem_limit: int | None = None
+    num_threads: int | None = None
+    scales: float | Sequence[float] | None = None
+    offsets: float | Sequence[float] | None = None
+    raster_creation_options: RasterCreationOptions | None = None
+
+    def __post_init__(self):
+        if self.dst_dtype == np.bool_:
+            self.dst_dtype = 'uint8'
+            self.nbits = 1
+
+    def __call__(self, dst_path: str | Path, spec: SpatialSpec) -> None:
+        """Reproject raster to specified spatial configuration.
+
+        Parameters
+        ----------
+        dst_path : str | Path
+            Destination file path
+        spec : SpatialSpec
+            Target spatial specification (CRS, transform, shape)
+        """
+        dst_path = Path(dst_path)
+
+        if dst_path.exists():
+            raise FileExistsError(f'Destination already exists: {dst_path}')
+
+        print(f'Reprojecting: {self.src_path} -> {dst_path}')
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            temp_path = Path(tmpdir) / f'~{"_".join(dst_path.parts[1:])}'
+
+            with rio.open(self.src_path) as src:
+                if len(src.subdatasets) > 1:
+                    sub_str = '\n'.join(src.subdatasets)
+                    raise RasterioIOError(
+                        f'Cannot reproject multi-variable dataset: {self.src_path}\nSubdatasets:\n{sub_str}',
+                    )
+
+                src_crs = src.crs if src.crs is not None else self.src_crs
+
+                if src_crs is None:
+                    raise ValueError(f'Cannot determine CRS for {self.src_path}. Provide src_crs parameter.')
+
+                src_dtype = src.dtypes[0]
+                src_transform = src.transform
+
+                src_nodata = src.nodata
+                if src_nodata is None and np.issubdtype(src_dtype, np.floating):
+                    src_nodata = np.nan
+
+                src_bands = src.indexes if self.bands is None else self.bands
+                if isinstance(src_bands, Number):
+                    src_bands = (src_bands,)
+                count = len(src_bands)
+
+                dst_dtype = src_dtype if self.dst_dtype is None else self.dst_dtype
+                dst_nodata = src_nodata if self.dst_nodata is None else self.dst_nodata
+
+                raster_creation_options = self.raster_creation_options or get_config().raster_creation_options
+
+                rio_profile = {
+                    'driver': 'GTiff',
+                    'height': spec.shape[0],
+                    'width': spec.shape[1],
+                    'dtype': dst_dtype,
+                    'nodata': dst_nodata,
+                    'count': count,
+                    'crs': spec.crs,
+                    'transform': spec.transform,
+                    **raster_creation_options.to_dict(),
+                }
+
+                if self.nbits is not None:
+                    rio_profile['nbits'] = self.nbits
+
+                warp_mem_limit = self.warp_mem_limit if self.warp_mem_limit is not None else get_config().warp_mem_limit
+                num_threads = self.num_threads if self.num_threads is not None else get_config().num_threads
+
+                with rio.open(temp_path, 'w', **rio_profile) as dst:
+                    rasterio.warp.reproject(
+                        source=rio.band(src, src_bands),
+                        destination=rio.band(dst, dst.indexes),
+                        src_crs=src_crs,
+                        dst_crs=spec.crs,
+                        src_transform=src_transform,
+                        dst_transform=spec.transform,
+                        src_nodata=src_nodata,
+                        dst_nodata=dst_nodata,
+                        resampling=self.resampling,
+                        warp_mem_limit=warp_mem_limit,
+                        num_threads=num_threads,
+                        **self.warp_kw,
+                    )
+
+                    scales = self.scales if self.scales is not None else tuple(src.scales[i - 1] for i in src_bands)
+                    offsets = self.offsets if self.offsets is not None else tuple(src.offsets[i - 1] for i in src_bands)
+
+                    if scales is not None:
+                        scales = scales if isinstance(scales, Sequence) else [scales] * dst.count
+                        dst._set_all_scales(scales)
+
+                    if offsets is not None:
+                        offsets = offsets if isinstance(offsets, Sequence) else [offsets] * dst.count
+                        dst._set_all_offsets(offsets)
+
+            shutil.move(temp_path, dst_path)
+
+    default_driver = RioXArrayDriver()
+    ext = 'tif'

pygeodata-0.0.0/pygeodata/types.py

@@ -0,0 +1,54 @@
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Protocol
+
+from affine import Affine
+from pyproj import CRS
+from rasterio.coords import BoundingBox
+
+RasterShape = tuple[int, int]
+
+
+@dataclass
+class SpatialSpec:
+    crs: CRS
+    transform: Affine
+    shape: RasterShape
+
+    @property
+    def resolution(self) -> tuple[int, int]:
+        return (abs(self.transform.a), abs(self.transform.e))
+
+    @property
+    def bounds(self) -> BoundingBox:
+        height, width = self.shape
+        x0, y0 = self.transform * (0, 0)
+        x1, y1 = self.transform * (width, height)
+        return BoundingBox(min(x0, x1), min(y0, y1), max(x0, x1), max(y0, y1))
+
+    @property
+    def extent(self) -> tuple[float, float, float, float]:
+        bounds = self.bounds
+        return (bounds.left, bounds.right, bounds.bottom, bounds.top)
+
+    def __repr__(self):
+        transform_str = (
+            f'Affine('
+            f'{self.transform.a:.2f}, '
+            f'{self.transform.b:.2f}, '
+            f'{self.transform.c:.2f}, '
+            f'{self.transform.d:.2f}, '
+            f'{self.transform.e:.2f}, '
+            f'{self.transform.f:.2f})'
+        )
+        return f'SpatialSpec(crs={self.crs.to_string()}, transform={transform_str}, shape={self.shape})'
+
+
+class Processor(Protocol):
+    def __call__(self, dst_path: str | Path, spec: SpatialSpec) -> None: ...
+
+
+class Driver(Protocol):
+    def __call__(self, path: str | Path) -> Any: ...
+
+    default_ext: str

pygeodata-0.0.0/pygeodata/utils.py

@@ -0,0 +1,5 @@
+from affine import Affine
+
+
+def transform_to_str(t: Affine) -> str:
+    return f'affine_{t.a:.4f}_{t.b:.4f}_{t.c:.4f}_{t.d:.4f}_{t.e:.4f}_{t.f:.4f}'

pygeodata-0.0.0/pygeodata.egg-info/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.4
+Name: pygeodata
+Version: 0.0.0
+Summary: Lightweight framework for geospatial data access and processing
+Author-email: Jasper Roebroek <roebroek.jasper@gmail.com>
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: rioxarray
+Requires-Dist: geopandas
+Requires-Dist: dask
+Requires-Dist: zarr
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-mock; extra == "test"
+Dynamic: license-file
+
+# pygeodata
+
+**Lightweight framework for geospatial data access and processing**
+
+`pygeodata` provides simple, composable tools for working with geospatial raster and vector data. It aims to make spatial data handling consistent, efficient, and compatible with large-scale and cloud-based workflows.
+
+## Features
+
+- Unified access to raster and vector datasets  
+- CRS-aware reprojection and alignment utilities
+- Clean, composable API designed for scientific workflows  
+
+## Installation
+
+```bash
+pip install pygeodata
+```

pygeodata-0.0.0/pygeodata.egg-info/SOURCES.txt

@@ -0,0 +1,23 @@
+LICENSE
+README.md
+pyproject.toml
+pygeodata/__init__.py
+pygeodata/base.py
+pygeodata/config.py
+pygeodata/loader.py
+pygeodata/options.py
+pygeodata/paths.py
+pygeodata/types.py
+pygeodata/utils.py
+pygeodata.egg-info/PKG-INFO
+pygeodata.egg-info/SOURCES.txt
+pygeodata.egg-info/dependency_links.txt
+pygeodata.egg-info/requires.txt
+pygeodata.egg-info/top_level.txt
+pygeodata/drivers/__init__.py
+pygeodata/drivers/rioxarray.py
+pygeodata/processors/__init__.py
+pygeodata/processors/rasterizer.py
+pygeodata/processors/reprojection.py
+tests/test_config.py
+tests/test_loader.py

pygeodata-0.0.0/pygeodata.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pygeodata-0.0.0/pygeodata.egg-info/requires.txt

@@ -0,0 +1,8 @@
+rioxarray
+geopandas
+dask
+zarr
+
+[test]
+pytest
+pytest-mock

pygeodata-0.0.0/pygeodata.egg-info/top_level.txt

@@ -0,0 +1 @@
+pygeodata

pygeodata-0.0.0/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pygeodata"
+description = "Lightweight framework for geospatial data access and processing"
+authors = [
+    { name = "Jasper Roebroek", email = "roebroek.jasper@gmail.com" }
+]
+dynamic = ["version"]
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.10"
+keywords = []
+classifiers = [
+    "Programming Language :: Python :: 3"
+]
+dependencies = [
+    "rioxarray",
+    "geopandas",
+    "dask",
+    "zarr"
+]
+
+[tool.setuptools.packages.find]
+include = ["pygeodata*"]
+
+[project.optional-dependencies]
+test = ["pytest", "pytest-mock"]

pygeodata-0.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pygeodata-0.0.0/tests/test_config.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+import pytest
+
+from pygeodata.config import Config, get_config, set_config
+
+
+def test_default_config():
+    cfg = get_config()
+    assert cfg.path_data_processed == Path('data_processed')
+
+
+def test_temporary_override():
+    original = get_config().path_data_processed
+    with set_config(path_data_processed=Path('/tmp')) as cfg:
+        assert cfg.path_data_processed == Path('/tmp')
+    assert get_config().path_data_processed == original
+
+
+def test_multiple_overrides():
+    with set_config(path_data_processed=Path('/tmp')) as cfg:
+        assert isinstance(cfg, Config)
+        assert cfg.path_data_processed == Path('/tmp')
+
+
+def test_config_invalid_key():
+    with pytest.raises(ValueError):
+        with set_config(invalid_key='value'):
+            pass

pygeodata-0.0.0/tests/test_loader.py

@@ -0,0 +1,282 @@
+from dataclasses import asdict
+
+from pygeodata import load
+from pygeodata.config import set_config
+from pygeodata.loader import DataLoader
+from pygeodata.types import SpatialSpec
+
+
+def test_loader_initialization(sample_loader_class):
+    """Test DataEntry initialization with default values."""
+    loader = sample_loader_class()
+    assert loader.name == 'sample'
+    assert hasattr(loader, 'processor')
+    assert hasattr(loader, 'driver')
+    assert loader.class_name == 'Sample'
+    assert loader.get_params() == {}
+
+
+def test_loader_get_processed_path(sample_loader_class, sample_spatial_spec, tmp_path):
+    loader = sample_loader_class()
+    with set_config(path_data_processed=tmp_path):
+        path = loader.get_processed_path(sample_spatial_spec)
+
+        assert path.parent.exists()
+        assert path.suffix == '.tif'
+        assert str(tmp_path) in str(path)
+
+
+def test_is_processed_false(sample_loader_class, sample_spatial_spec, tmp_path):
+    """Test is_processed when file doesn't exist."""
+    with set_config(path_data_processed=tmp_path):
+        assert not sample_loader_class().is_processed(sample_spatial_spec)
+
+
+def test_is_processed_true(sample_loader_class, sample_spatial_spec, tmp_path):
+    """Test is_processed when file exists."""
+    loader = sample_loader_class()
+    with set_config(path_data_processed=tmp_path):
+        path = loader.get_processed_path(sample_spatial_spec)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.touch()
+
+        assert loader.is_processed(sample_spatial_spec)
+
+
+def test_process_creates_file(sample_loader_class, sample_spatial_spec, tmp_path):
+    """Test that process creates the expected output file."""
+    loader = sample_loader_class()
+    with set_config(path_data_processed=tmp_path):
+        loader.process(sample_spatial_spec)
+        path = loader.get_processed_path(sample_spatial_spec)
+        assert path.exists()
+
+
+def test_load_with_params(sample_loader_class_complex, sample_spatial_spec):
+    """Test that parameters are included in the generated path."""
+    assert set(sample_loader_class_complex(10, 10).get_params()) == {'time', 'resolution'}
+
+
+def test_loader_repr(sample_loader_class):
+    """Test string representation of loader."""
+    loader = sample_loader_class()
+    repr_str = repr(loader)
+    assert repr_str.startswith('Sample(')
+    assert repr_str.endswith(')')
+
+
+def test_loader_repr_with_params(sample_loader_class_complex):
+    """Test string representation includes parameters sorted alphabetically."""
+    loader = sample_loader_class_complex(time=10, resolution=5)
+    repr_str = repr(loader)
+    assert 'ComplexSample(' in repr_str
+    assert 'resolution=5' in repr_str
+    assert 'time=10' in repr_str
+    assert repr_str.index('resolution') < repr_str.index('time')
+
+
+def test_name_conversion_simple(sample_loader_class):
+    """Test class name to snake_case conversion for simple names."""
+    loader = sample_loader_class()
+    assert loader.name == 'sample'
+
+
+def test_name_conversion_camel_case():
+    """Test class name conversion handles camelCase properly."""
+
+    class MyDataLoader(DataLoader):
+        @property
+        def processor(self):
+            return lambda path, spec: None
+
+    loader = MyDataLoader()
+    assert loader.name == 'my_data'
+
+
+def test_name_conversion_acronyms():
+    """Test class name conversion handles acronyms properly."""
+
+    class XMLHTTPLoader(DataLoader):
+        @property
+        def processor(self):
+            return lambda path, spec: None
+
+    loader = XMLHTTPLoader()
+    assert loader.name == 'xmlhttp'
+
+
+def test_name_conversion_mixed():
+    """Test class name conversion handles mixed acronyms and camelCase."""
+
+    class USGSElevationLoader(DataLoader):
+        @property
+        def processor(self):
+            return lambda path, spec: None
+
+    loader = USGSElevationLoader()
+    assert loader.name == 'usgs_elevation'
+
+
+def test_get_params_excludes_private_attributes(sample_loader_class_complex):
+    """Test that get_params excludes private attributes."""
+    loader = sample_loader_class_complex(time=10, resolution=5)
+    loader._private_attr = 'should_not_appear'
+    params = loader.get_params()
+    assert '_private_attr' not in params
+
+
+def test_get_params_multiple_params(sample_loader_class_complex):
+    """Test get_params returns all instance attributes."""
+    loader = sample_loader_class_complex(time=15, resolution=20)
+    params = loader.get_params()
+    assert params['time'] == 15
+    assert params['resolution'] == 20
+    assert len(params) == 2
+
+
+def test_process_skips_if_already_processed(sample_loader_class, sample_spatial_spec, tmp_path):
+    """Test that process doesn't reprocess if file already exists."""
+    loader = sample_loader_class()
+    with set_config(path_data_processed=tmp_path):
+        path = loader.get_processed_path(sample_spatial_spec)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.touch()
+
+        loader.is_processed(sample_spatial_spec)
+
+
+def test_process_calls_processor_if_not_processed(sample_loader_class, sample_spatial_spec, tmp_path, mocker):
+    """Test that process calls processor when file doesn't exist."""
+    loader = sample_loader_class()
+    with set_config(path_data_processed=tmp_path):
+        # Mock the processor
+        mock_processor = mocker.MagicMock()
+        mocker.patch.object(loader, 'processor', mock_processor)
+
+        loader.process(sample_spatial_spec)
+
+        path = loader.get_processed_path(sample_spatial_spec)
+        mock_processor.assert_called_once_with(path, sample_spatial_spec)
+
+
+def test_load_returns_data(sample_loader_class, sample_spatial_spec, tmp_path, mocker):
+    """Test that load returns data from driver."""
+    loader = sample_loader_class()
+    expected_data = {'test': 'data'}
+
+    with set_config(path_data_processed=tmp_path):
+        # Mock driver.load to return test data
+        mock_load = mocker.MagicMock(return_value=expected_data)
+        mocker.patch.object(loader, 'load', mock_load)
+
+        result = loader.load(sample_spatial_spec)
+
+        assert result == expected_data
+        mock_load.assert_called_once()
+
+
+def test_load_processes_before_loading(sample_loader_class, sample_spatial_spec, tmp_path, mocker):
+    """Test that load calls process before loading data."""
+    loader = sample_loader_class()
+
+    with set_config(path_data_processed=tmp_path):
+        mock_process = mocker.patch.object(loader, 'process')
+        mock_load = mocker.patch.object(loader, 'load', return_value=None)
+
+        loader(sample_spatial_spec)
+
+        # Ensure process is called before load
+        mock_process.assert_called_once_with(sample_spatial_spec)
+        mock_load.assert_called_once()
+
+
+def test_is_processed_with_valid_symlink(sample_loader_class, sample_spatial_spec, tmp_path):
+    """Test is_processed returns True for valid symlink."""
+    loader = sample_loader_class()
+    with set_config(path_data_processed=tmp_path):
+        path = loader.get_processed_path(sample_spatial_spec)
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Create a target file and symlink to it
+        target = tmp_path / 'target.tif'
+        target.touch()
+        path.symlink_to(target)
+
+        assert loader.is_processed(sample_spatial_spec)
+
+
+def test_is_processed_with_broken_symlink(sample_loader_class, sample_spatial_spec, tmp_path):
+    """Test is_processed behavior with broken symlink."""
+    loader = sample_loader_class()
+    with set_config(path_data_processed=tmp_path):
+        path = loader.get_processed_path(sample_spatial_spec)
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Create a symlink to non-existent target
+        target = tmp_path / 'nonexistent.tif'
+        path.symlink_to(target)
+
+        assert not loader.is_processed(sample_spatial_spec)
+
+
+def test_driver_fallback_to_processor_default(sample_loader_class):
+    """Test that driver property falls back to processor's default_driver."""
+    loader = sample_loader_class()
+    driver = loader.driver
+    assert driver is not None
+    assert hasattr(driver, '__call__')
+
+
+def test_get_processed_path_creates_parent_directory(sample_loader_class, sample_spatial_spec, tmp_path):
+    """Test that get_processed_path creates parent directories."""
+    loader = sample_loader_class()
+    with set_config(path_data_processed=tmp_path):
+        path = loader.get_processed_path(sample_spatial_spec)
+        # Parent should be created by get_processed_path
+        assert path.parent.exists()
+
+
+def test_multiple_specs_different_paths(sample_loader_class, sample_spatial_spec, tmp_path):
+    """Test that different spatial specs generate different paths."""
+    loader = sample_loader_class()
+
+    # Create a second spec with different properties
+    spec2 = SpatialSpec(**asdict(sample_spatial_spec))
+    spec2.shape = (2, 3)
+
+    with set_config(path_data_processed=tmp_path):
+        path1 = loader.get_processed_path(sample_spatial_spec)
+        path2 = loader.get_processed_path(spec2)
+
+        assert path1 != path2
+
+
+def test_same_spec_same_path(sample_loader_class, sample_spatial_spec, tmp_path):
+    """Test that same spec always generates the same path."""
+    loader = sample_loader_class()
+
+    with set_config(path_data_processed=tmp_path):
+        path1 = loader.get_processed_path(sample_spatial_spec)
+        path2 = loader.get_processed_path(sample_spatial_spec)
+
+        assert path1 == path2
+
+
+def test_different_loader_instances_same_behavior(sample_loader_class, sample_spatial_spec, tmp_path):
+    """Test that different instances of the same loader class behave identically."""
+    loader1 = sample_loader_class()
+    loader2 = sample_loader_class()
+
+    with set_config(path_data_processed=tmp_path):
+        path1 = loader1.get_processed_path(sample_spatial_spec)
+        path2 = loader2.get_processed_path(sample_spatial_spec)
+
+        assert path1 == path2
+        assert loader1.name == loader2.name
+        assert loader1.class_name == loader2.class_name
+
+
+def test_load(sample_loader_class, sample_spatial_spec, tmp_path):
+    with set_config(path_data_processed=tmp_path):
+        data = load(sample_loader_class(), spec=sample_spatial_spec)
+        assert data.rio.crs == sample_spatial_spec.crs