yirgacheffe 1.2.0__py3-none-any.whl

yirgacheffe/layers/h3layer.py

@@ -0,0 +1,203 @@
+from math import ceil, floor
+from typing import Any
+
+import h3
+import numpy as np
+from osgeo import gdal
+
+from ..rounding import round_up_pixels
+from ..window import Area, PixelScale, Window
+from .base import YirgacheffeLayer
+from ..backends import backend
+
+class H3CellLayer(YirgacheffeLayer):
+
+    def __init__(self, cell_id: str, scale: PixelScale, projection: str):
+        if not h3.is_valid_cell(cell_id):
+            raise ValueError(f"{cell_id} is not a valid H3 cell identifier")
+        self.cell_id = cell_id
+        self.zoom = h3.get_resolution(cell_id)
+
+        self.cell_boundary = h3.cell_to_boundary(cell_id)
+
+        abs_xstep, abs_ystep = abs(scale.xstep), abs(scale.ystep)
+        area = Area(
+            left=(floor(min(x[1] for x in self.cell_boundary) / abs_xstep) * abs_xstep),
+            top=(ceil(max(x[0] for x in self.cell_boundary) / abs_ystep) * abs_ystep),
+            right=(ceil(max(x[1] for x in self.cell_boundary) / abs_xstep) * abs_xstep),
+            bottom=(floor(min(x[0] for x in self.cell_boundary) / abs_ystep) * abs_ystep),
+        )
+
+        # Statistically, most hex tiles will be within a projection, but some will wrap around
+        # the edge, so check if we're hit one of those cases.
+        self.centre = h3.cell_to_latlng(cell_id)
+
+        # Due to time constraints, and that most geospatial data we are working with stops before the poles
+        # I'm going to explicitly not handle the poles right now, where you get different distortions from
+        # at the 180 line:
+        if not area.bottom < self.centre[0] < area.top:
+            raise NotImplementedError("Distortion at poles not currently handled")
+
+        # For wrap around due to hitting the longitunal wrap, we currently just do the naive thing and
+        # project a band for the full width of the projection. I have a plan to fix this, as I'd like layers
+        # to have sublayers, allowing us to handle vector layers more efficiently, but curently we have a
+        # deadline, and given how infrequently we hit this case, doing the naive thing for now is sufficient
+        if (abs(area.left - area.right)) > 180.0:
+            left = (-180.0 / abs_xstep) * abs_xstep
+            if left < -180.0:
+                left += abs_xstep
+            right = (180.0 / abs_xstep) * abs_xstep
+            if right > 180.0:
+                right -= abs_xstep
+            area = Area(
+                left=left,
+                right=right,
+                top=area.top,
+                bottom=area.bottom,
+            )
+
+        super().__init__(area, scale, projection)
+        self._window = Window(
+            xoff=0,
+            yoff=0,
+            xsize=round_up_pixels((area.right - area.left) / scale.xstep, abs_xstep),
+            ysize=round_up_pixels((area.bottom - area.top) / scale.ystep, abs_ystep),
+        )
+        self._raster_xsize = self.window.xsize
+        self._raster_ysize = self.window.ysize
+
+        # see comment in read_array for why we're doing this
+        sorted_lats = [x[0] for x in self.cell_boundary]
+        sorted_lats.sort()
+        self._raster_safe_bounds = (
+            (sorted_lats[-2] / abs_ystep) * abs_ystep,
+            (sorted_lats[1] / abs_ystep) * abs_ystep,
+        )
+
+
+    @property
+    def datatype(self) -> int:
+        return gdal.GDT_Float64
+
+    def read_array_with_window(self, xoffset: int, yoffset: int, xsize: int, ysize: int, window: Window) -> Any:
+        if (xsize <= 0) or (ysize <= 0):
+            raise ValueError("Request dimensions must be positive and non-zero")
+
+        # We have two paths: one for the common case where the hex cell doesn't cross 180˚ longitude,
+        # and another case for where it does
+        max_width_projection = self.area.right - self.area.left
+        if max_width_projection < 180:
+
+            target_window = Window(
+                window.xoff + xoffset,
+                window.yoff + yoffset,
+                xsize,
+                ysize
+            )
+            source_window = Window(
+                xoff=0,
+                yoff=0,
+                xsize=self._raster_xsize,
+                ysize=self._raster_ysize,
+            )
+            try:
+                intersection = Window.find_intersection([source_window, target_window])
+            except ValueError:
+                return 0.0
+
+            subset = np.zeros((intersection.ysize, intersection.xsize))
+
+            start_x = self._active_area.left + ((intersection.xoff - self.window.xoff) * self._pixel_scale.xstep)
+            start_y = self._active_area.top + ((intersection.yoff - self.window.yoff) * self._pixel_scale.ystep)
+
+            for ypixel in range(intersection.ysize):
+                # The latlng_to_cell is quite expensive, so ideally we want to avoid
+                # calling latlng_to_cell for every pixel, though I do want to do that
+                # rather than infer from the cell_boundary, as this way I'm ensured
+                # I get an authoritative result and no pixel falls between the cracks.
+
+                # Originally I just tried the old-school way of finding the left-most
+                # and the right-most pixel per row, and then filling between, which in
+                # theory should work fine for rasterising hexagons, but in certain places
+                # the map projection distorts the edges sufficiently they become concave,
+                # and that approach lead to over filling, which caused issues with cells
+                # overlapping.
+
+                # The current implementation tries to hedge its bets by looking where the
+                # lowest and highest edges are (stored in _raster_safe_bounds) and then
+                # only doing the fill between left and right within those bounds. I suspect
+                # longer term this will need some padding, but it works for the current set
+                # of known failures in test_optimisation. As we hit more issues we should
+                # expand that test case before tweaking here. This is quite wasteful, as it's
+                # only a tiny number of cells that have convex edges, so in future it'd be
+                # interesting to see if we can infer when that is.
+
+                lat = start_y + (ypixel * self._pixel_scale.ystep)
+
+                if self._raster_safe_bounds[0] < lat < self._raster_safe_bounds[1]:
+                    # In "safe" zone, try to be clever
+                    left_most = intersection.xsize + 1
+                    right_most = -1
+
+                    for xpixel in range(intersection.xsize):
+                        lng = start_x + (xpixel * self._pixel_scale.xstep)
+                        this_cell = h3.latlng_to_cell(lat, lng, self.zoom)
+                        if this_cell == self.cell_id:
+                            left_most = xpixel
+                            break
+
+                    for xpixel in range(intersection.xsize - 1, left_most - 1, -1):
+                        lng = start_x + (xpixel * self._pixel_scale.xstep)
+                        this_cell = h3.latlng_to_cell(lat, lng, self.zoom)
+                        if this_cell == self.cell_id:
+                            right_most = xpixel
+                            break
+
+                    for xpixel in range(left_most, right_most + 1, 1):
+                        subset[ypixel][xpixel] = 1.0
+                else:
+                    # Not in safe zone, be diligent.
+                    for xpixel in range(intersection.xsize):
+                        lng = start_x + (xpixel * self._pixel_scale.xstep)
+                        this_cell = h3.latlng_to_cell(lat, lng, self.zoom)
+                        if this_cell == self.cell_id:
+                            subset[ypixel][xpixel] = 1.0
+
+            return backend.pad(
+                backend.promote(subset),
+                (
+                    (
+                        (intersection.yoff - self.window.yoff) - yoffset,
+                        (ysize - ((intersection.yoff - self.window.yoff) + intersection.ysize)) + yoffset,
+                    ),
+                    (
+                        (intersection.xoff - self.window.xoff) - xoffset,
+                        xsize - ((intersection.xoff - self.window.xoff) + intersection.xsize) + xoffset,
+                    )
+                ),
+                'constant'
+            )
+        else:
+            # This handles the case where the cell wraps over 180˚ longitude
+            res = np.zeros((ysize, xsize))
+
+            left = min(x[1] for x in self.cell_boundary if x[1] > 0.0)
+            right = max(x[1] for x in self.cell_boundary if x[1] < 0.0) + 360.0
+            max_width_projection = right - left
+            max_width = ceil(max_width_projection / self._pixel_scale.xstep)
+
+            for ypixel in range(yoffset, yoffset + ysize):
+                lat = self._active_area.top + (ypixel * self._pixel_scale.ystep)
+
+                for xpixel in range(xoffset, min(xoffset + xsize, max_width)):
+                    lng = self._active_area.left + (xpixel * self._pixel_scale.xstep)
+                    this_cell = h3.latlng_to_cell(lat, lng, self.zoom)
+                    if this_cell == self.cell_id:
+                        res[ypixel - yoffset][xpixel - xoffset] = 1.0
+
+                for xpixel in range(xoffset + xsize - 1, xoffset + xsize - max_width, -1):
+                    lng = self._active_area.left + (xpixel * self._pixel_scale.xstep)
+                    this_cell = h3.latlng_to_cell(lat, lng, self.zoom)
+                    if this_cell == self.cell_id:
+                        res[ypixel - yoffset][xpixel - xoffset] = 1.0
+            return backend.promote(res)

yirgacheffe/layers/rasters.py

@@ -0,0 +1,333 @@
+import math
+import os
+from typing import Any, Optional, TypeVar, Union
+
+import numpy as np
+from osgeo import gdal
+
+from .. import WGS_84_PROJECTION
+from ..window import Area, PixelScale, Window
+from ..rounding import round_up_pixels
+from .base import YirgacheffeLayer
+from ..backends import backend
+
+# Still to early to require Python 3.11 :/
+RasterLayerT = TypeVar("RasterLayerT", bound="RasterLayer")
+
+class InvalidRasterBand(Exception):
+    def __init__ (self, band):
+        self.band = band
+
+class RasterLayer(YirgacheffeLayer):
+    """Layer provides a wrapper around a gdal dataset/band that also records offset state so that
+    we can work with maps over different geographic regions but work withing a particular frame
+    of reference."""
+
+    @staticmethod
+    def empty_raster_layer(
+        area: Area,
+        scale: PixelScale,
+        datatype: int,
+        filename: Optional[str]=None,
+        projection: str=WGS_84_PROJECTION,
+        name: Optional[str]=None,
+        compress: bool=True,
+        nodata: Optional[Union[float,int]]=None,
+        nbits: Optional[int]=None,
+        threads: Optional[int]=None,
+        bands: int=1
+    ) -> RasterLayerT:
+        abs_xstep, abs_ystep = abs(scale.xstep), abs(scale.ystep)
+
+        # We treat the provided area as aspirational, and we need to align it to pixel boundaries
+        pixel_friendly_area = Area(
+            left=math.floor(area.left / abs_xstep) * abs_xstep,
+            right=math.ceil(area.right / abs_xstep) * abs_xstep,
+            top=math.ceil(area.top / abs_ystep) * abs_ystep,
+            bottom=math.floor(area.bottom / abs_ystep) * abs_ystep,
+        )
+
+        options = []
+        if threads is not None:
+            options.append(f"NUM_THREADS={threads}")
+        if nbits is not None:
+            options.append(f"NBITS={nbits}")
+
+        if filename:
+            driver = gdal.GetDriverByName('GTiff')
+            options.append('BIGTIFF=YES')
+            if compress:
+                options.append('COMPRESS=LZW')
+            else:
+                options.append('COMPRESS=NONE')
+        else:
+            driver = gdal.GetDriverByName('mem')
+            filename = 'mem'
+            compress = False
+        dataset = driver.Create(
+            filename,
+            round_up_pixels((pixel_friendly_area.right - pixel_friendly_area.left) / abs_xstep, abs_xstep),
+            round_up_pixels((pixel_friendly_area.top - pixel_friendly_area.bottom) / abs_ystep, abs_ystep),
+            bands,
+            datatype,
+            options
+        )
+        dataset.SetGeoTransform([
+            pixel_friendly_area.left, scale.xstep, 0.0, pixel_friendly_area.top, 0.0, scale.ystep
+        ])
+        dataset.SetProjection(projection)
+        if nodata is not None:
+            dataset.GetRasterBand(1).SetNoDataValue(nodata)
+        return RasterLayer(dataset, name=name)
+
+    @staticmethod
+    def empty_raster_layer_like(
+        layer: Any,
+        filename: Optional[str]=None,
+        area: Optional[Area]=None,
+        datatype: Optional[int]=None,
+        compress: bool=True,
+        nodata: Optional[Union[float,int]]=None,
+        nbits: Optional[int]=None,
+        threads: Optional[int]=None,
+        bands: int=1
+    ) -> RasterLayerT:
+        width = layer.window.xsize
+        height = layer.window.ysize
+        if area is None:
+            area = layer.area
+        assert area is not None
+
+        scale = layer.pixel_scale
+        if scale is None:
+            raise ValueError("Can not work out area without explicit pixel scale")
+        abs_xstep, abs_ystep = abs(scale.xstep), abs(scale.ystep)
+        width = round_up_pixels((area.right - area.left) / abs_xstep, abs_xstep)
+        height = round_up_pixels((area.top - area.bottom) / abs_ystep, abs_ystep)
+        geo_transform = (
+            area.left, scale.xstep, 0.0, area.top, 0.0, scale.ystep
+        )
+
+        options = []
+        if threads is not None:
+            options.append(f"NUM_THREADS={threads}")
+        if nbits is not None:
+            options.append(f"NBITS={nbits}")
+
+        if filename:
+            driver = gdal.GetDriverByName('GTiff')
+            options.append('BIGTIFF=YES')
+            if compress:
+                options.append('COMPRESS=LZW')
+            else:
+                options.append('COMPRESS=NONE')
+        else:
+            driver = gdal.GetDriverByName('mem')
+            filename = 'mem'
+            compress = False
+        dataset = driver.Create(
+            filename,
+            width,
+            height,
+            bands,
+            datatype if datatype is not None else layer.datatype,
+            options,
+        )
+        dataset.SetGeoTransform(geo_transform)
+        dataset.SetProjection(layer.projection)
+        if nodata is not None:
+            dataset.GetRasterBand(1).SetNoDataValue(nodata)
+
+        return RasterLayer(dataset)
+
+    @classmethod
+    def scaled_raster_from_raster(
+        cls,
+        source: RasterLayerT,
+        new_pixel_scale: PixelScale,
+        filename: Optional[str]=None,
+        compress: bool=True,
+        algorithm: int=gdal.GRA_NearestNeighbour,
+    ) -> RasterLayerT:
+        source_dataset = source._dataset
+        old_pixel_scale = source.pixel_scale
+        assert old_pixel_scale
+
+        x_scale = old_pixel_scale.xstep / new_pixel_scale.xstep
+        y_scale = old_pixel_scale.ystep / new_pixel_scale.ystep
+        new_width = round_up_pixels(source_dataset.RasterXSize * x_scale,
+            abs(new_pixel_scale.xstep))
+        new_height = round_up_pixels(source_dataset.RasterYSize * y_scale,
+            abs(new_pixel_scale.ystep))
+
+        # in yirgacheffe we like to have things aligned to the pixel_scale, so work
+        # out new top left corner
+        new_left = math.floor((source.area.left / new_pixel_scale.xstep)) * new_pixel_scale.xstep
+        new_top = math.ceil((source.area.top / new_pixel_scale.ystep)) * new_pixel_scale.ystep
+
+        # now build a target dataset
+        options = []
+        if filename:
+            driver = gdal.GetDriverByName('GTiff')
+            options.append('BIGTIFF=YES')
+            if compress:
+                options.append('COMPRESS=LZW')
+        else:
+            driver = gdal.GetDriverByName('mem')
+            filename = 'mem'
+            compress = False
+        dataset = driver.Create(
+            filename,
+            new_width,
+            new_height,
+            1,
+            source.datatype,
+            options
+        )
+        dataset.SetGeoTransform((
+            new_left, new_pixel_scale.xstep, 0.0,
+            new_top, 0.0, new_pixel_scale.ystep
+        ))
+        dataset.SetProjection(source_dataset.GetProjection())
+
+        # now use gdal to do the reprojection
+        gdal.ReprojectImage(source_dataset, dataset, eResampleAlg=algorithm)
+
+        return RasterLayer(dataset)
+
+    @classmethod
+    def layer_from_file(cls, filename: str, band: int = 1) -> RasterLayerT:
+        try:
+            dataset = gdal.Open(filename, gdal.GA_ReadOnly)
+        except RuntimeError as exc:
+            # With exceptions on GDAL now returns the wrong (IMHO) exception
+            raise FileNotFoundError(filename) from exc
+        try:
+            _ = dataset.GetRasterBand(band)
+        except RuntimeError as exc:
+            raise InvalidRasterBand(band) from exc
+        return cls(dataset, filename, band)
+
+    def __init__(self, dataset: gdal.Dataset, name: Optional[str] = None, band: int = 1):
+        if not dataset:
+            raise ValueError("None is not a valid dataset")
+
+        transform = dataset.GetGeoTransform()
+        scale = PixelScale(transform[1], transform[5])
+        area = Area(
+            left=transform[0],
+            top=transform[3],
+            right=transform[0] + (dataset.RasterXSize * scale.xstep),
+            bottom=transform[3] + (dataset.RasterYSize * scale.ystep),
+        )
+
+        super().__init__(
+            area,
+            scale,
+            dataset.GetProjection(),
+            name=name
+        )
+
+        # The constructor works out the window from the area
+        # so sanity check that the calculated window matches the
+        # dataset's dimensions
+        assert self.window == Window(0, 0, dataset.RasterXSize, dataset.RasterYSize)
+
+        self._dataset = dataset
+        self._dataset_path = dataset.GetDescription()
+        self._band = band
+        self._raster_xsize = dataset.RasterXSize
+        self._raster_ysize = dataset.RasterYSize
+
+    def close(self):
+        try:
+            if self._dataset:
+                try:
+                    self._dataset.Close()
+                except AttributeError:
+                    pass
+                del self._dataset
+        except AttributeError:
+            # Don't error if close was already called
+            pass
+
+    def __getstate__(self) -> object:
+        # Only support pickling on file backed layers (ideally read only ones...)
+        if not os.path.isfile(self._dataset_path):
+            raise ValueError("Can not pickle layer that is not file backed.")
+        odict = self.__dict__.copy()
+        self._park()
+        del odict['_dataset']
+        return odict
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+        self._unpark()
+
+    def _park(self):
+        try:
+            self._dataset.Close()
+        except AttributeError:
+            pass
+        self._dataset = None
+
+    def _unpark(self):
+        if getattr(self, "_dataset", None) is None:
+            try:
+                self._dataset = gdal.Open(self._dataset_path)
+            except RuntimeError as exc:
+                raise FileNotFoundError(f"Failed to open pickled raster {self._dataset_path}") from exc
+
+    @property
+    def datatype(self) -> int:
+        if self._dataset is None:
+            self._unpark()
+        assert self._dataset
+        return self._dataset.GetRasterBand(1).DataType
+
+    def read_array_with_window(self, xoffset: int, yoffset: int, xsize: int, ysize: int, window: Window) -> Any:
+        if self._dataset is None:
+            self._unpark()
+        assert self._dataset
+
+        if (xsize <= 0) or (ysize <= 0):
+            raise ValueError("Request dimensions must be positive and non-zero")
+
+        # if we're dealing with an intersection, we can just read the data directly,
+        # otherwise we need to read the data into another array with suitable padding
+        target_window = Window(
+            window.xoff + xoffset,
+            window.yoff + yoffset,
+            xsize,
+            ysize
+        )
+        source_window = Window(
+            xoff=0,
+            yoff=0,
+            xsize=self._raster_xsize,
+            ysize=self._raster_ysize,
+        )
+        try:
+            intersection = Window.find_intersection([source_window, target_window])
+        except ValueError:
+            return backend.zeros((ysize, xsize))
+
+        if target_window == intersection:
+            # The target window is a subset of or equal to the source, so we can just ask for the data
+            data = backend.promote(self._dataset.GetRasterBand(self._band).ReadAsArray(*intersection.as_array_args))
+            return data
+        else:
+            # We should read the intersection from the array, and the rest should be zeros
+            subset = backend.promote(self._dataset.GetRasterBand(self._band).ReadAsArray(*intersection.as_array_args))
+            region = np.array((
+                (
+                    (intersection.yoff - window.yoff) - yoffset,
+                    (ysize - ((intersection.yoff - window.yoff) + intersection.ysize)) + yoffset,
+                ),
+                (
+                    (intersection.xoff - window.xoff) - xoffset,
+                    xsize - ((intersection.xoff - window.xoff) + intersection.xsize) + xoffset,
+                )
+            )).astype(int)
+            data = backend.pad(subset, region, mode='constant')
+            return data

yirgacheffe/layers/rescaled.py

@@ -0,0 +1,94 @@
+from math import floor, ceil
+from typing import Any, Optional
+
+from skimage import transform
+
+from ..window import PixelScale, Window
+from .rasters import RasterLayer, YirgacheffeLayer
+from ..backends import backend
+
+
+class RescaledRasterLayer(YirgacheffeLayer):
+    """RescaledRaster dynamically rescales a raster, so to you to work with multiple layers at
+    different scales without having to store unnecessary data. """
+
+    @classmethod
+    def layer_from_file(
+        cls,
+        filename: str,
+        pixel_scale: PixelScale,
+        band: int = 1,
+        nearest_neighbour: bool = True,
+    ):
+        src = RasterLayer.layer_from_file(filename, band=band)
+        return RescaledRasterLayer(src, pixel_scale, nearest_neighbour, src.name)
+
+    def __init__(
+        self,
+        src: RasterLayer,
+        pixel_scale: PixelScale,
+        nearest_neighbour: bool = True,
+        name: Optional[str] = None,
+    ):
+        super().__init__(
+            src.area,
+            pixel_scale=pixel_scale,
+            projection=src.projection,
+            name=name
+        )
+
+        self._src = src
+        self._nearest_neighbour = nearest_neighbour
+
+        src_pixel_scale = src.pixel_scale
+        assert src_pixel_scale # from raster we should always have one
+
+        self._x_scale = src_pixel_scale.xstep / pixel_scale.xstep
+        self._y_scale = src_pixel_scale.ystep / pixel_scale.ystep
+
+    def close(self):
+        self._src.close()
+
+    def _park(self):
+        self._src._park()
+
+    def _unpark(self):
+        self._src._unpark()
+
+    @property
+    def datatype(self) -> int:
+        return self._src.datatype
+
+    def read_array_with_window(self, xoffset: int, yoffset: int, xsize: int, ysize: int, window: Window) -> Any:
+
+        # to avoid aliasing issues, we try to scale to the nearest pixel
+        # and recrop when scaling bigger
+
+        xoffset = xoffset + window.xoff
+        yoffset = yoffset + window.yoff
+
+        src_x_offset = floor(xoffset / self._x_scale)
+        src_y_offset = floor(yoffset / self._y_scale)
+
+        diff_x = floor(((xoffset / self._x_scale) - src_x_offset) * self._x_scale)
+        diff_y = floor(((yoffset / self._x_scale) - src_y_offset) * self._x_scale)
+
+        src_x_width = ceil((xsize + diff_x) / self._x_scale)
+        src_y_width = ceil((ysize + diff_y) / self._y_scale)
+
+        # Get the matching src data
+        src_data = backend.demote_array(self._src.read_array(
+            src_x_offset,
+            src_y_offset,
+            src_x_width,
+            src_y_width
+        ))
+
+        scaled = transform.resize(
+            src_data,
+            (src_y_width * self._y_scale, src_x_width * self._x_scale),
+            order=(0 if self._nearest_neighbour else 1),
+            anti_aliasing=(not self._nearest_neighbour)
+        )
+
+        return backend.promote(scaled[diff_y:(diff_y + ysize),diff_x:(diff_x + xsize)])