yirgacheffe 1.9.4__py3-none-any.whl → 1.10.0__py3-none-any.whl

yirgacheffe/__init__.py

@@ -13,7 +13,7 @@ except ModuleNotFoundError:
     __version__ = pyproject_data["project"]["version"]
 
 from .layers import YirgacheffeLayer
-from ._core import read_raster, read_rasters, read_shape, read_shape_like, constant, read_narrow_raster
+from ._core import read_raster, read_rasters, read_shape, read_shape_like, constant, read_narrow_raster, from_array
 from .constants import WGS_84_PROJECTION
 from .window import Area, MapProjection, Window
 from ._backends.enumeration import dtype as DataType

yirgacheffe/_backends/enumeration.py

@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 from enum import Enum
 
+import numpy as np
 from osgeo import gdal
 
 class operators(Enum):
@@ -39,6 +42,13 @@ class operators(Enum):
     ROUND = 34
     CEIL = 35
     ISNAN = 36
+    RADD = 37
+    RSUB = 38
+    RMUL = 39
+    RTRUEDIV = 40
+    RFLOORDIV = 41
+    RREMAINDER = 42
+    RPOW = 43
 
 class dtype(Enum):
     """Represents the type of data returned by a layer.
@@ -77,5 +87,31 @@ class dtype(Enum):
         return self.value
 
     @classmethod
-    def of_gdal(cls, val):
+    def of_gdal(cls, val: int) -> dtype:
         return cls(val)
+
+    @classmethod
+    def of_array(cls, val: np.ndarray) -> dtype:
+        match val.dtype:
+            case np.float32:
+                return dtype.Float32
+            case np.float64:
+                return dtype.Float64
+            case np.int8:
+                return dtype.Int8
+            case np.int16:
+                return dtype.Int16
+            case np.int32:
+                return dtype.Int32
+            case np.int64:
+                return dtype.Int64
+            case np.uint8:
+                return dtype.UInt8
+            case np.uint16:
+                return dtype.UInt16
+            case np.uint32:
+                return dtype.UInt32
+            case np.uint64:
+                return dtype.UInt64
+            case _:
+                raise ValueError

yirgacheffe/_backends/mlx.py

@@ -222,4 +222,11 @@ operator_map: dict[op, Callable] = {
     op.ROUND: mx.round,
     op.CEIL: mx.ceil,
     op.ISNAN: mx.isnan,
+    op.RADD: mx.array.__radd__,
+    op.RSUB: mx.array.__rsub__,
+    op.RMUL: mx.array.__rmul__,
+    op.RTRUEDIV: mx.array.__rtruediv__,
+    op.RFLOORDIV: mx.array.__rfloordiv__,
+    op.RREMAINDER: mx.array.__rmod__,
+    op.RPOW: mx.array.__rpow__,
 }

yirgacheffe/_backends/numpy.py

@@ -160,4 +160,15 @@ operator_map: dict[op, Callable] = {
     op.ROUND: np.round,
     op.CEIL: np.ceil,
     op.ISNAN: np.isnan,
+    # Pylint doesn't recognise the reverse operators, despite them
+    # clearly existing on np.ndarray, so we have to disable the checks
+    # pylint: disable=no-member
+    op.RADD: np.ndarray.__radd__,
+    op.RSUB: np.ndarray.__rsub__,
+    op.RMUL: np.ndarray.__rmul__,
+    op.RTRUEDIV: np.ndarray.__rtruediv__,
+    op.RFLOORDIV: np.ndarray.__rfloordiv__,
+    op.RREMAINDER: np.ndarray.__rmod__,
+    op.RPOW: np.ndarray.__rpow__,
+    # pylint: enable=no-member
 }

yirgacheffe/_core.py

@@ -3,13 +3,15 @@ from __future__ import annotations
 from pathlib import Path
 from typing import Sequence
 
+import numpy as np
+
 from .layers.area import UniformAreaLayer
 from .layers.base import YirgacheffeLayer
 from .layers.constant import ConstantLayer
 from .layers.group import GroupLayer, TiledGroupLayer
 from .layers.rasters import RasterLayer
 from .layers.vectors import VectorLayer
-from .window import MapProjection
+from .window import Area, MapProjection
 from ._backends.enumeration import dtype as DataType
 
 def read_raster(
@@ -161,3 +163,50 @@ def constant(value: int | float) -> YirgacheffeLayer:
         A constant layer of the provided value.
     """
     return ConstantLayer(value)
+
+def from_array(
+    values: np.ndarray,
+    origin: tuple[float, float],
+    projection: MapProjection | tuple[str, tuple[float, float]],
+) -> YirgacheffeLayer:
+    """Creates an in-memory layer from a numerical array.
+
+    Args:
+        values: a 2D array of data values, with Y on the first dimension, X on
+            the second dimension.
+        origin: the position of the top left pixel in the geospatial space
+        projection: the map projection and pixel scale to use.
+
+    Returns:
+        A geospatial layer that uses the provided data for its values.
+    """
+
+    if projection is None:
+        raise ValueError("Projection must not be none")
+
+    if not isinstance(projection, MapProjection):
+        projection_name, scale_tuple = projection
+        projection = MapProjection(projection_name, scale_tuple[0], scale_tuple[1])
+
+    dims = values.shape
+
+    area = Area(
+        left=origin[0],
+        top=origin[1],
+        right=origin[0] + (projection.xstep * dims[1]),
+        bottom=origin[1] + (projection.ystep * dims[0])
+    )
+
+    layer = RasterLayer.empty_raster_layer(
+        area,
+        scale=projection.scale,
+        datatype=DataType.of_array(values),
+        filename=None,
+        projection=projection.name,
+    )
+    assert layer._dataset
+    assert layer._dataset.RasterXSize == dims[1]
+    assert layer._dataset.RasterYSize == dims[0]
+    layer._dataset.GetRasterBand(1).WriteArray(values, 0, 0)
+
+    return layer

yirgacheffe/{_operators.py → _operators/__init__.py}

@@ -13,9 +13,11 @@ import types
 from collections.abc import Callable
 from contextlib import ExitStack
 from enum import Enum
-from multiprocessing import Semaphore, Process
+from multiprocessing import Process, Semaphore
+from multiprocessing.synchronize import Semaphore as SemaphoreType
 from multiprocessing.managers import SharedMemoryManager
 from pathlib import Path
+from typing import Any
 
 import deprecation
 import numpy as np
@@ -24,14 +26,15 @@ from osgeo import gdal
 from dill import dumps, loads # type: ignore
 from pyproj import Transformer
 
-from . import constants, __version__
-from .rounding import round_up_pixels, round_down_pixels
-from .window import Area, PixelScale, MapProjection, Window
-from ._backends import backend
-from ._backends.enumeration import operators as op
-from ._backends.enumeration import dtype as DataType
-from ._backends.numpy import dtype_to_backend as dtype_to_numpy
-from ._backends.numpy import backend_to_dtype as numpy_to_dtype
+from .. import constants, __version__
+from ..rounding import round_up_pixels, round_down_pixels
+from ..window import Area, PixelScale, MapProjection, Window
+from .._backends import backend
+from .._backends.enumeration import operators as op
+from .._backends.enumeration import dtype as DataType
+from .._backends.numpy import dtype_to_backend as dtype_to_numpy
+from .._backends.numpy import backend_to_dtype as numpy_to_dtype
+from .cse import CSECacheTable
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.WARNING)
@@ -50,9 +53,13 @@ class LayerConstant:
     def __str__(self) -> str:
         return str(self.val)
 
-    def _eval(self, _area, _projection, _index, _step, _target_window):
+    def _eval(self, _cse_cache, _area, _projection, _index, _step, _target_window):
         return self.val
 
+    @property
+    def _cse_hash(self) -> int | None:
+        return hash(self.val)
+
     @property
     def datatype(self) -> DataType:
         numpy_type = np.result_type(self.val)
@@ -70,24 +77,45 @@ class LayerMathMixin:
     def __add__(self, other):
         return LayerOperation(self, op.ADD, other, window_op=WindowOperation.UNION)
 
+    def __radd__(self, other):
+        return LayerOperation(self, op.RADD, other, window_op=WindowOperation.UNION)
+
     def __sub__(self, other):
         return LayerOperation(self, op.SUB, other, window_op=WindowOperation.UNION)
 
+    def __rsub__(self, other):
+        return LayerOperation(self, op.RSUB, other, window_op=WindowOperation.UNION)
+
     def __mul__(self, other):
         return LayerOperation(self, op.MUL, other, window_op=WindowOperation.INTERSECTION)
 
+    def __rmul__(self, other):
+        return LayerOperation(self, op.RMUL, other, window_op=WindowOperation.INTERSECTION)
+
     def __truediv__(self, other):
         return LayerOperation(self, op.TRUEDIV, other, window_op=WindowOperation.INTERSECTION)
 
+    def __rtruediv__(self, other):
+        return LayerOperation(self, op.RTRUEDIV, other, window_op=WindowOperation.INTERSECTION)
+
     def __floordiv__(self, other):
         return LayerOperation(self, op.FLOORDIV, other, window_op=WindowOperation.INTERSECTION)
 
+    def __rfloordiv__(self, other):
+        return LayerOperation(self, op.RFLOORDIV, other, window_op=WindowOperation.INTERSECTION)
+
     def __mod__(self, other):
         return LayerOperation(self, op.REMAINDER, other, window_op=WindowOperation.INTERSECTION)
 
+    def __rmod__(self, other):
+        return LayerOperation(self, op.RREMAINDER, other, window_op=WindowOperation.INTERSECTION)
+
     def __pow__(self, other):
         return LayerOperation(self, op.POW, other, window_op=WindowOperation.UNION)
 
+    def __rpow__(self, other):
+        return LayerOperation(self, op.RPOW, other, window_op=WindowOperation.UNION)
+
     def __eq__(self, other):
         return LayerOperation(self, op.EQ, other, window_op=WindowOperation.INTERSECTION)
 
@@ -114,17 +142,22 @@ class LayerMathMixin:
 
     def _eval(
         self,
-        area,
-        projection,
-        index,
-        step,
-        target_window=None
+        cse_cache: CSECacheTable,
+        area: Area,
+        projection: MapProjection,
+        index: int,
+        step: int,
+        target_window: Window,
     ):
+        cache_data = cse_cache.get_data(self._cse_hash, target_window)
+        if cache_data is not None:
+            return cache_data
+
         try:
             window = self.window if target_window is None else target_window
-            return self._read_array_for_area(area, projection, 0, index, window.xsize, step)
+            result = self._read_array_for_area(area, projection, 0, index, window.xsize, step)
         except AttributeError:
-            return self._read_array_for_area(
+            result = self._read_array_for_area(
                 area,
                 projection,
                 0,
@@ -133,6 +166,9 @@ class LayerMathMixin:
                 step
             )
 
+        cse_cache.set_data(self._cse_hash, target_window, result)
+        return result
+
     def nan_to_num(self, nan=0, posinf=None, neginf=None):
         return LayerOperation(
             self,
@@ -149,7 +185,7 @@ class LayerMathMixin:
             self,
             op.ISIN,
             window_op=WindowOperation.NONE,
-            test_elements=test_elements,
+            test_elements=tuple(test_elements),
         )
 
     def isnan(self):
@@ -252,10 +288,10 @@ class LayerMathMixin:
             weights=weights.astype(np.float32),
         )
 
-    def numpy_apply(self, func, other=None):
+    def numpy_apply(self, func: Callable, other=None):
         return LayerOperation(self, func, other)
 
-    def shader_apply(self, func, other=None):
+    def shader_apply(self, func: Callable, other=None):
         return ShaderStyleOperation(self, func, other)
 
     def save(self, destination_layer, and_sum=False, callback=None, band=1):
@@ -347,9 +383,16 @@ class LayerMathMixin:
     def area(self) -> Area:
         raise NotImplementedError("Must be overridden by subclass")
 
+    @property
+    def _cse_hash(self) -> int | None:
+        raise NotImplementedError("Must be overridden by subclass")
+
     def read_array(self, _x, _y, _w, _h):
         raise NotImplementedError("Must be overridden by subclass")
 
+    def _read_array_for_area(self, _target_area, _target_projection, _x, _y, _w, _h):
+        raise NotImplementedError("Must be overridden by subclass")
+
     def show(self, ax=None, max_pixels: int | None =1000, **kwargs):
         """Display data using matplotlib.
 
@@ -424,12 +467,12 @@ class LayerOperation(LayerMathMixin):
 
     def __init__(
         self,
-        lhs,
-        operator=None,
-        rhs=None,
-        other=None,
-        window_op=WindowOperation.NONE,
-        buffer_padding=0,
+        lhs: Any,
+        operator: op | Callable | None = None,
+        rhs: Any = None,
+        other: Any = None,
+        window_op: WindowOperation = WindowOperation.NONE,
+        buffer_padding: int = 0,
         **kwargs
     ):
         self.ystep = constants.YSTEP
@@ -446,7 +489,7 @@ class LayerOperation(LayerMathMixin):
 
         if rhs is not None:
             if backend.isscalar(rhs):
-                self.rhs = LayerConstant(rhs)
+                self.rhs: Any = LayerConstant(rhs)
             elif isinstance(rhs, (backend.array_t)):
                 if rhs.shape == ():
                     self.rhs = LayerConstant(rhs.item())
@@ -461,7 +504,7 @@ class LayerOperation(LayerMathMixin):
 
         if other is not None:
             if backend.isscalar(other):
-                self.other = LayerConstant(other)
+                self.other: Any = LayerConstant(other)
             elif isinstance(other, (backend.array_t)):
                 if other.shape == ():
                     self.rhs = LayerConstant(other.item())
@@ -474,6 +517,13 @@ class LayerOperation(LayerMathMixin):
         else:
             self.other = None
 
+        # this is expensive, so don't do it all the time
+        self._cse_hash_cache = self._calc_cse_hash()
+
+    @property
+    def _cse_hash(self) -> int | None:
+        return self._cse_hash_cache
+
     def __str__(self) -> str:
         try:
             return f"({self.lhs} {self.operator} {self.rhs})"
@@ -483,9 +533,6 @@ class LayerOperation(LayerMathMixin):
             except AttributeError:
                 return str(self.lhs)
 
-    def __len__(self) -> int:
-        return len(self.lhs)
-
     def __getstate__(self) -> object:
         odict = self.__dict__.copy()
         if isinstance(self.operator, types.LambdaType):
@@ -499,6 +546,35 @@ class LayerOperation(LayerMathMixin):
             del state['operator_dill']
         self.__dict__.update(state)
 
+    @property
+    def _children(self) -> list:
+        return [x for x in [self.lhs, self.rhs, self.other] if x is not None]
+
+    def _calc_cse_hash(self) -> int | None:
+        # If we can't hash any of the child nodes then we can't hash this node, as if we can't store
+        # their results in the cache, we can't know this result is stable
+        child_hashes = [x._cse_hash for x in self._children]
+        if any(x is None for x in child_hashes):
+            return None
+
+        # This really should be recursive
+        def _make_hashable(value):
+            if isinstance(value, (list, tuple, set)):
+                return tuple(value)
+            if isinstance(value, np.ndarray):
+                return id(value)
+            else:
+                return value
+        frozen_kwargs = tuple(sorted((k, _make_hashable(v)) for (k, v) in self.kwargs.items()))
+
+        terms = [self.operator, self.window_op, frozen_kwargs, self.buffer_padding] + child_hashes
+
+        try:
+            return hash(tuple(terms))
+        except TypeError:
+            # This is assumed to be because kwargs contains something unhashable
+            return None
+
     @property
     def area(self) -> Area:
         return self._get_operation_area(self.map_projection)
@@ -546,7 +622,7 @@ class LayerOperation(LayerMathMixin):
                 )
                 return union
             case _:
-                assert False, "Should not be reached"
+                raise RuntimeError("Should not be reached")
 
     @property
     @deprecation.deprecated(
@@ -632,15 +708,35 @@ class LayerOperation(LayerMathMixin):
                 pass
         return projection
 
+    def pretty_print(self, prefix="", is_last=True):
+        kwargs_str = ", ".join(f"{k}={v}" for k, v in self.kwargs.items())
+        label = f"{self.operator}({kwargs_str})" if kwargs_str else self.operator
+        label = f"{label} - {self._cse_hash}"
+
+        connector = "└── " if is_last else "├── "
+        print(f"{prefix}{connector}{label}")
+
+        extension = "    " if is_last else "│   "
+        new_prefix = prefix + extension
+
+        children = self._children
+        for i, child in enumerate(children):
+            child_is_last = i == len(children) - 1
+            if hasattr(child, 'pretty_print'):
+                child.pretty_print(new_prefix, child_is_last)
+            else:
+                child_connector = "└── " if child_is_last else "├── "
+                print(f"{new_prefix}{child_connector}{repr(child)}")
+
     def _eval(
         self,
+        cse_cache: CSECacheTable,
         area: Area,
         projection: MapProjection,
         index: int,
         step: int,
-        target_window: Window | None = None
+        target_window: Window,
     ):
-
         if self.buffer_padding:
             if target_window:
                 target_window = target_window.grow(self.buffer_padding)
@@ -648,30 +744,36 @@ class LayerOperation(LayerMathMixin):
             # The index doesn't need updating because we updated area/window
             step += (2 * self.buffer_padding)
 
-        lhs_data = self.lhs._eval(area, projection, index, step, target_window)
-
-        if self.operator is None:
-            return lhs_data
-
-        try:
-            operator: Callable = backend.operator_map[self.operator]
-        except KeyError:
-            # Handles things like `numpy_apply` where a custom operator is provided
-            operator = self.operator
+        cache_data = cse_cache.get_data(self._cse_hash, target_window)
+        if cache_data is not None:
+            return cache_data
 
-        if self.other is not None:
-            assert self.rhs is not None
-            rhs_data = self.rhs._eval(area, projection, index, step, target_window)
-            other_data = self.other._eval(area, projection, index, step, target_window)
-            return operator(lhs_data, rhs_data, other_data, **self.kwargs)
+        lhs_data = self.lhs._eval(cse_cache, area, projection, index, step, target_window)
 
-        if self.rhs is not None:
-            rhs_data = self.rhs._eval(area, projection, index, step, target_window)
-            return operator(lhs_data, rhs_data, **self.kwargs)
+        if self.operator is None:
+            result = lhs_data
+        else:
+            if isinstance(self.operator, op):
+                operator = backend.operator_map[self.operator]
+            else:
+                # Handles things like `numpy_apply` where a custom operator is provided
+                operator = self.operator
+
+            if self.other is not None:
+                assert self.rhs is not None
+                rhs_data = self.rhs._eval(cse_cache, area, projection, index, step, target_window)
+                other_data = self.other._eval(cse_cache, area, projection, index, step, target_window)
+                result = operator(lhs_data, rhs_data, other_data, **self.kwargs)
+            elif self.rhs is not None:
+                rhs_data = self.rhs._eval(cse_cache, area, projection, index, step, target_window)
+                result = operator(lhs_data, rhs_data, **self.kwargs)
+            else:
+                result = operator(lhs_data, **self.kwargs)
 
-        return operator(lhs_data, **self.kwargs)
+        cse_cache.set_data(self._cse_hash, target_window, result)
+        return result
 
-    def sum(self):
+    def sum(self) -> float:
         # The result accumulator is float64, and for precision reasons
         # we force the sum to be done in float64 also. Otherwise we
         # see variable results depending on chunk size, as different parts
@@ -679,40 +781,77 @@ class LayerOperation(LayerMathMixin):
         res = 0.0
         computation_window = self.window
         projection = self.map_projection
+        if projection is None:
+            raise ValueError("No map projection")
+
+        cse_cache = CSECacheTable(self, computation_window)
+
         for yoffset in range(0, computation_window.ysize, self.ystep):
+            cse_cache.reset_cache()
             step=self.ystep
             if yoffset+step > computation_window.ysize:
                 step = computation_window.ysize - yoffset
-            chunk = self._eval(self._get_operation_area(projection), projection, yoffset, step, computation_window)
+            chunk = self._eval(
+                cse_cache,
+                self._get_operation_area(projection),
+                projection,
+                yoffset,
+                step,
+                computation_window
+            )
             res += backend.sum_op(chunk)
         return res
 
-    def min(self):
-        res = None
+    def min(self) -> float:
+        res = float('inf')
         computation_window = self.window
         projection = self.map_projection
+        if projection is None:
+            raise ValueError("No map projection")
+
+        cse_cache = CSECacheTable(self, computation_window)
+
         for yoffset in range(0, computation_window.ysize, self.ystep):
+            cse_cache.reset_cache()
             step=self.ystep
             if yoffset+step > computation_window.ysize:
                 step = computation_window.ysize - yoffset
-            chunk = self._eval(self._get_operation_area(projection), projection, yoffset, step, computation_window)
-            chunk_min = backend.min_op(chunk)
-            if (res is None) or (res > chunk_min):
-                res = chunk_min
+            chunk = self._eval(
+                cse_cache,
+                self._get_operation_area(projection),
+                projection,
+                yoffset,
+                step,
+                computation_window
+            )
+            chunk_min = float(backend.min_op(chunk))
+            res = min(res, chunk_min)
         return res
 
-    def max(self):
-        res = None
+    def max(self) -> float:
+        res = float('-inf')
         computation_window = self.window
         projection = self.map_projection
+        if projection is None:
+            raise ValueError("No map projection")
+
+        cse_cache = CSECacheTable(self, computation_window)
+
         for yoffset in range(0, computation_window.ysize, self.ystep):
+            cse_cache.reset_cache()
             step=self.ystep
             if yoffset+step > computation_window.ysize:
                 step = computation_window.ysize - yoffset
-            chunk = self._eval(self._get_operation_area(projection), projection, yoffset, step, computation_window)
-            chunk_max = backend.max_op(chunk)
-            if (res is None) or (chunk_max > res):
-                res = chunk_max
+            chunk = self._eval(
+                cse_cache,
+                self._get_operation_area(projection),
+                projection,
+                yoffset,
+                step,
+                computation_window
+            )
+            chunk_max = float(backend.max_op(chunk))
+            res = max(res, chunk_max)
         return res
 
     def save(self, destination_layer, and_sum=False, callback=None, band=1) -> float | None:
@@ -757,13 +896,18 @@ class LayerOperation(LayerMathMixin):
 
         total = 0.0
 
+        cse_cache = CSECacheTable(self, computation_window)
+
         for yoffset in range(0, computation_window.ysize, self.ystep):
+
+            cse_cache.reset_cache()
+
             if callback:
                 callback(yoffset / computation_window.ysize)
             step = self.ystep
             if yoffset + step > computation_window.ysize:
                 step = computation_window.ysize - yoffset
-            chunk = self._eval(computation_area, projection, yoffset, step, computation_window)
+            chunk = self._eval(cse_cache, computation_area, projection, yoffset, step, computation_window)
             if isinstance(chunk, (float, int)):
                 chunk = backend.full((step, destination_window.xsize), chunk)
             band.WriteArray(
@@ -778,11 +922,29 @@ class LayerOperation(LayerMathMixin):
 
         return total if and_sum else None
 
-    def _parallel_worker(self, index, shared_mem, sem, np_dtype, width, input_queue, output_queue, computation_window):
-        arr = np.ndarray((self.ystep, width), dtype=np_dtype, buffer=shared_mem.buf)
+    def _parallel_worker(
+        self,
+        index : int,
+        shared_mem,
+        sem : SemaphoreType,
+        np_dtype : type,
+        width : int,
+        input_queue : multiprocessing.Queue,
+        output_queue : multiprocessing.Queue,
+        computation_window : Window,
+    ):
+        # The hashing of python objects isn't stable across processes in general, so we have to do
+        # the cache build once per worker
+        cse_cache = CSECacheTable(self, computation_window)
+
+        arr = np.ndarray((self.ystep, width), dtype=np_dtype, buffer=shared_mem.buf) # type: ignore[var-annotated]
         projection = self.map_projection
+        # TODO: the `save` method does more sanity checking that parallel save!
+        assert projection is not None
         try:
             while True:
+                cse_cache.reset_cache()
+
                 # We acquire the lock so we know we have somewhere to put the
                 # result before we take work. This is because in practice
                 # it seems the writing to GeoTIFF is the bottleneck, and
@@ -798,7 +960,14 @@ class LayerOperation(LayerMathMixin):
                     break
                 yoffset, step = task
 
-                result = self._eval(self._get_operation_area(projection), projection, yoffset, step, computation_window)
+                result = self._eval(
+                    cse_cache,
+                    self._get_operation_area(projection),
+                    projection,
+                    yoffset,
+                    step,
+                    computation_window,
+                )
                 backend.eval_op(result)
 
                 arr[:step] = backend.demote_array(result)
@@ -844,7 +1013,7 @@ class LayerOperation(LayerMathMixin):
             elif and_sum:
                 return self.sum()
             else:
-                assert False
+                raise RuntimeError("Should not be reached") # pylint: disable=W0707
 
         worker_count = parallelism or multiprocessing.cpu_count()
         work_blocks = len(range(0, computation_window.ysize, self.ystep))
@@ -857,7 +1026,7 @@ class LayerOperation(LayerMathMixin):
             elif and_sum:
                 return self.sum()
             else:
-                assert False
+                raise RuntimeError("Should not be reached")
 
         if destination_layer is not None:
             try:
@@ -942,7 +1111,7 @@ class LayerOperation(LayerMathMixin):
                         computation_window.xsize,
                         source_queue,
                         result_queue,
-                        computation_window
+                        computation_window,
                     )) for i in range(worker_count)]
                     for worker in workers:
                         worker.start()
@@ -1074,11 +1243,15 @@ class LayerOperation(LayerMathMixin):
         )
 
         chunks = []
+
+        cse_cache = CSECacheTable(self, computation_window)
+
         for yoffset in range(0, height, self.ystep):
+            cse_cache.reset_cache()
             step = self.ystep
             if yoffset + step > height:
                 step = height - yoffset
-            chunk = self._eval(computation_area, projection, yoffset, step, computation_window)
+            chunk = self._eval(cse_cache, computation_area, projection, yoffset, step, computation_window)
             if isinstance(chunk, (float, int)):
                 chunk = backend.full((step, computation_window.xsize), chunk)
             chunks.append(chunk)
@@ -1086,14 +1259,17 @@ class LayerOperation(LayerMathMixin):
 
         return res
 
+    def _read_array_for_area(self, _target_area, _target_projection, _x, _y, _w, _h):
+        raise RuntimeError("Should not be called")
+
 class ShaderStyleOperation(LayerOperation):
 
-    def _eval(self, area, projection, index, step, target_window=None):
+    def _eval(self, cse_cache, area, projection, index, step, target_window=None):
         if target_window is None:
             target_window = self.window
-        lhs_data = self.lhs._eval(area, projection, index, step, target_window)
+        lhs_data = self.lhs._eval(cse_cache, area, projection, index, step, target_window)
         if self.rhs is not None:
-            rhs_data = self.rhs._eval(area, projection, index, step, target_window)
+            rhs_data = self.rhs._eval(cse_cache, area, projection, index, step, target_window)
         else:
             rhs_data = None
 
@@ -1127,6 +1303,9 @@ class ShaderStyleOperation(LayerOperation):
 
         return result
 
+    def _read_array_for_area(self, _target_area, _target_projection, _x, _y, _w, _h):
+        raise RuntimeError("Should not be called")
+
 def where(cond, a, b):
     """Return elements chosen from `a` or `b` depending on `cond`.
 

yirgacheffe/_operators/cse.py

@@ -0,0 +1,66 @@
+
+from .._backends import backend
+from ..window import Window
+
+class CSECacheTable:
+
+    def __init__(self, expression, window: Window) -> None:
+        self._table: dict[tuple[int, Window], tuple[int, backend.array_t | None]] = {}
+        self._populate(expression, window)
+
+    def __len__(self) -> int:
+        return len(self._table)
+
+    def _add(self, cse_hash: int | None, window: Window) -> int:
+        if cse_hash is None:
+            return 0
+
+        try:
+            count, data = self._table[(cse_hash, window)]
+            cache_line = (count + 1, data)
+        except KeyError:
+            cache_line = (1, None)
+        self._table[(cse_hash, window)] = cache_line
+        count, _ = cache_line
+        return count
+
+    def _populate(self, expression, window: Window) -> None:
+        used_window = window.grow(expression.buffer_padding)
+        count = self._add(expression._cse_hash, used_window)
+        if count == 1:
+            # We only add children the first time we see an expression, otherwise
+            # we will cache data covered by this node's cache line potentially
+            for child in expression._children:
+                try:
+                    self._populate(child, used_window)
+                except AttributeError:
+                    try:
+                        self._add(child._cse_hash, used_window)
+                    except TypeError:
+                        pass
+
+    def set_data(self, cse_hash: int | None, window: Window, data: backend.array_t) -> None:
+        if cse_hash is not None:
+            try:
+                count, old_data = self._table[(cse_hash, window)]
+                if count < 2:
+                    return
+                if old_data is not None:
+                    raise RuntimeWarning("Failure in CSE logic, setting data that is already set")
+                self._table[(cse_hash, window)] = (count, data)
+            except KeyError:
+                raise RuntimeWarning("Failure in CSE logic, setting data for unknown term") # pylint: disable=W0707
+
+    def get_data(self, cse_hash: int | None, window: Window) -> backend.array_t | None:
+        # TODO: In theory we could release data from the table if we decremented the count each read
+        # but we'd also need to fixed copy of the count for reset
+        if cse_hash is None:
+            return None
+        try:
+            _count, data = self._table[(cse_hash, window)]
+            return data
+        except KeyError:
+            return None
+
+    def reset_cache(self):
+        self._table = {k: (count, None) for k, (count, _data) in self._table.items()}

yirgacheffe/layers/base.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+import uuid
 from typing import Any, Sequence
 
 import deprecation
@@ -28,7 +29,7 @@ class YirgacheffeLayer(LayerMathMixin):
         self._active_area: Area | None = None
         self._projection = projection
         self._window: Window | None = None
-        self.name = name
+        self.name = name if name is not None else str(uuid.uuid4())
 
         self.reset_window()
 
@@ -41,6 +42,10 @@ class YirgacheffeLayer(LayerMathMixin):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
+    @property
+    def _cse_hash(self) -> int | None:
+        raise NotImplementedError("Must be overridden by subclass")
+
     def _park(self) -> None:
         pass
 

yirgacheffe/layers/constant.py

@@ -15,6 +15,10 @@ class ConstantLayer(YirgacheffeLayer):
         super().__init__(area, None)
         self.value = float(value)
 
+    @property
+    def _cse_hash(self) -> int | None:
+        return hash(self.value)
+
     @property
     def datatype(self) -> DataType:
         return DataType.Float64

yirgacheffe/layers/group.py

@@ -72,6 +72,10 @@ class GroupLayer(YirgacheffeLayer):
         self._underlying_layers.reverse()
         self.layers = self._underlying_layers
 
+    @property
+    def _cse_hash(self) -> int | None:
+        return hash(tuple(x._cse_hash for x in self._underlying_layers))
+
     def _park(self) -> None:
         for layer in self.layers:
             try:
@@ -81,7 +85,7 @@ class GroupLayer(YirgacheffeLayer):
 
     @property
     def datatype(self) -> DataType:
-        return DataType.of_gdal(self.layers[0].datatype)
+        return self.layers[0].datatype
 
     def set_window_for_intersection(self, new_area: Area) -> None:
         super().set_window_for_intersection(new_area)

yirgacheffe/layers/h3layer.py

@@ -76,6 +76,15 @@ class H3CellLayer(YirgacheffeLayer):
             (sorted_lats[1] / abs_ystep) * abs_ystep,
         )
 
+    @property
+    def _cse_hash(self) -> int | None:
+        return hash((
+            self.cell_id,
+            self._underlying_area,
+            self.map_projection,
+            self._active_area,
+        ))
+
     @property
     def _raster_dimensions(self) -> tuple[int, int]:
         return (self._raster_xsize, self._raster_ysize)

yirgacheffe/layers/rasters.py

@@ -313,6 +313,18 @@ class RasterLayer(YirgacheffeLayer):
             except RuntimeError as exc:
                 raise FileNotFoundError(f"Failed to open pickled raster {self._dataset_path}") from exc
 
+    @property
+    def _cse_hash(self) -> int | None:
+        return hash((
+            self.name,
+            self._underlying_area,
+            self.map_projection,
+            self.datatype,
+            self._active_area,
+            self._ignore_nodata,
+            self._band,
+        ))
+
     @property
     def datatype(self) -> DataType:
         if self._dataset is None:

yirgacheffe/layers/rescaled.py

@@ -52,6 +52,17 @@ class RescaledRasterLayer(YirgacheffeLayer):
         self._x_scale = src_projection.xstep / target_projection.xstep
         self._y_scale = src_projection.ystep / target_projection.ystep
 
+    @property
+    def _cse_hash(self) -> int | None:
+        return hash((
+            self._src._cse_hash,
+            self.name,
+            self._underlying_area,
+            self._nearest_neighbour,
+            self.map_projection,
+            self._active_area
+        ))
+
     def close(self):
         self._src.close()
 

yirgacheffe/layers/vectors.py

@@ -303,7 +303,6 @@ class VectorLayer(YirgacheffeLayer):
         if layer is None:
             raise ValueError('No layer provided')
         self.layer = layer
-        self.name = name
 
         if isinstance(datatype, int):
             self._datatype = DataType.of_gdal(datatype)
@@ -364,7 +363,7 @@ class VectorLayer(YirgacheffeLayer):
                 bottom=floor(min(x[2] for x in envelopes)),
             )
 
-        super().__init__(area, projection)
+        super().__init__(area, projection, name=name)
 
 
     def _get_operation_area(self, projection: MapProjection | None = None) -> Area:
@@ -441,6 +440,18 @@ class VectorLayer(YirgacheffeLayer):
             if self._filter is not None:
                 self.layer.SetAttributeFilter(self._filter)
 
+    @property
+    def _cse_hash(self) -> int | None:
+        return hash((
+            self.name,
+            self._underlying_area,
+            self.map_projection,
+            self._active_area,
+            self._datatype,
+            self.burn_value,
+            self._filter,
+        ))
+
     @property
     def datatype(self) -> DataType:
         return self._datatype

yirgacheffe/window.py

@@ -22,6 +22,15 @@ class MapProjection:
         name: The map projection used in WKT format.
         xstep: The number of units horizontal distance a step of one pixel makes in the map projection.
         ystep: The number of units vertical distance a step of one pixel makes in the map projection.
+
+    Examples:
+        Create a map projection using an EPSG code:
+
+        >>> proj_wgs84 = MapProjection("epsg:4326", 0.001, -0.001)
+
+        Create a projection using an ESRI code:
+
+        >>> proj_esri = MapProjection("esri:54030", 1000, -1000)
     """
 
     def __init__(self, projection_string: str, xstep: float, ystep: float) -> None:
@@ -32,6 +41,9 @@ class MapProjection:
         self.xstep = xstep
         self.ystep = ystep
 
+    def __hash__(self):
+        return hash((self.name, self.xstep, self.ystep))
+
     def __eq__(self, other) -> bool:
         if other is None:
             return True
@@ -52,7 +64,7 @@ class MapProjection:
     def scale(self) -> PixelScale:
         return PixelScale(self.xstep, self.ystep)
 
-@dataclass
+@dataclass(frozen=True)
 class Area:
     """Class to hold a geospatial area of data in the given projection.
 
@@ -145,7 +157,7 @@ class Area:
             (other.top >= self.bottom >= other.bottom)
         )
 
-@dataclass
+@dataclass(frozen=True)
 class Window:
     """Class to hold the pixel dimensions of data in the given projection.
 
@@ -216,7 +228,7 @@ class Window:
         """
         return Window(
             xoff=self.xoff - pixels,
-            yoff=self.xoff - pixels,
+            yoff=self.yoff - pixels,
             xsize=self.xsize + (2 * pixels),
             ysize=self.ysize + (2 * pixels),
         )

{yirgacheffe-1.9.4.dist-info → yirgacheffe-1.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.9.4
+Version: 1.10.0
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC
@@ -37,6 +37,7 @@ Requires-Dist: mypy; extra == "dev"
 Requires-Dist: pylint; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
 Requires-Dist: build; extra == "dev"
 Requires-Dist: twine; extra == "dev"
 Requires-Dist: mkdocs-material; extra == "dev"

yirgacheffe-1.10.0.dist-info/RECORD

@@ -0,0 +1,28 @@
+yirgacheffe/__init__.py,sha256=Ps6W8A1TRriVNxZEF3jW1_KOLEtji4ffyoGRmQXne8g,927
+yirgacheffe/_core.py,sha256=Tr6RAiRZOO3vbtiTjLoNRjjd1DkXYZOgpDqrjs7jCBw,7231
+yirgacheffe/constants.py,sha256=bKUjOGNj19zwggV79lJgK7tiv51DH2-rgNOKswl2gvQ,293
+yirgacheffe/operators.py,sha256=Y1KkNt79N1elR4ZplQaQngx29wdf2QFF_5la4PI3EhI,412
+yirgacheffe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+yirgacheffe/rounding.py,sha256=Jzd9qlLnLpigT95GbQTByvYOo639Nfq4LBEVyvhYdoc,2289
+yirgacheffe/window.py,sha256=11YqUw5GZs8qRjHSgQrvPbapTNFpx6OGwlj0qUtQc70,9863
+yirgacheffe/_backends/__init__.py,sha256=jN-2iRrHStnPI6cNL7XhwhsROtI0EaGfIrbF5c-ECV0,334
+yirgacheffe/_backends/enumeration.py,sha256=xeAWNfMORHd4Ue-CryajmteD-nxjIGcrU5m9PdXgEqA,2765
+yirgacheffe/_backends/mlx.py,sha256=0Kq2weyC_fo-luATcMU1fBKB3aJjAg5Bl2_A3Mejp4E,6451
+yirgacheffe/_backends/numpy.py,sha256=zgZo2u9ZVW8LYRKtLzSkFz3w7Y1j3_KwvVHTZ_xqGpE,4605
+yirgacheffe/_operators/__init__.py,sha256=UZEy-0FR6un3UOMdws3XzqdaaDQLiWy0IqV9hOh2baE,49977
+yirgacheffe/_operators/cse.py,sha256=4x5kwAGbTe8nBZYipz53mpW-Cr81A4vK-v6YWq5SmKg,2602
+yirgacheffe/layers/__init__.py,sha256=mYKjw5YTcMNv_hMy7a6K4yRzIuNUbR8WuBTw4WIAmSk,435
+yirgacheffe/layers/area.py,sha256=wJcMHbLJBaXS4BeFbu5rYeKfgu3gvaE9hwQ5j6aw-y4,3976
+yirgacheffe/layers/base.py,sha256=jqtlAbRSHWDMZpEOl0dVuNNYKzR90s7uEuECvXCA3Dk,13378
+yirgacheffe/layers/constant.py,sha256=eheddNT3fjMDpfISRipqKzOLJxR01D1IMerGJpperls,1567
+yirgacheffe/layers/group.py,sha256=R0g-PhZprweVRYBJD5hp5trM8ztMCi0yyK-3UhDPGoo,16266
+yirgacheffe/layers/h3layer.py,sha256=X5lfuM-TKVly6cWn9B-VbMTijmlsi7kUgp9OTYnJzJ0,10128
+yirgacheffe/layers/rasters.py,sha256=Q9MPmS9WYJ17_inpL4KVkjJBX-jizjvERPdb8Mq-r5o,13820
+yirgacheffe/layers/rescaled.py,sha256=7QzHL0l73-fS3XAhkdBRb3lhYJ-HnLrpjTByVfei9Fg,3630
+yirgacheffe/layers/vectors.py,sha256=KxVUgBIgNyFuACyGkcdK8eU09stVCf5VQAvBL7RfoGA,20268
+yirgacheffe-1.10.0.dist-info/licenses/LICENSE,sha256=dNSHwUCJr6axStTKDEdnJtfmDdFqlE3h1NPCveqPfnY,757
+yirgacheffe-1.10.0.dist-info/METADATA,sha256=rRgnXn19ivS7CUPT78xw9RGoh0h1Nmm80JuPuAsVG9g,7451
+yirgacheffe-1.10.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+yirgacheffe-1.10.0.dist-info/entry_points.txt,sha256=j4KgHXbVGbGyfTySc1ypBdERpfihO4WNjppvCdE9HjE,52
+yirgacheffe-1.10.0.dist-info/top_level.txt,sha256=9DBFlKO2Ld3hG6TuE3qOTd3Tt8ugTiXil4AN4Wr9_y0,12
+yirgacheffe-1.10.0.dist-info/RECORD,,

yirgacheffe-1.9.4.dist-info/RECORD

@@ -1,27 +0,0 @@
-yirgacheffe/__init__.py,sha256=JWQOJP_RPlBmN_N3vPGH-9FByjRQKCWUSnsh4fPNtW8,915
-yirgacheffe/_core.py,sha256=-X6wTjAKagcjWcH14IKp75Nf-kANO_Nsd8wKm5XEQzE,5750
-yirgacheffe/_operators.py,sha256=VGQ9AOOJP6cxsr_G2kxdaPaXqKi7K1csocxsudlRwVc,43440
-yirgacheffe/constants.py,sha256=bKUjOGNj19zwggV79lJgK7tiv51DH2-rgNOKswl2gvQ,293
-yirgacheffe/operators.py,sha256=Y1KkNt79N1elR4ZplQaQngx29wdf2QFF_5la4PI3EhI,412
-yirgacheffe/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-yirgacheffe/rounding.py,sha256=Jzd9qlLnLpigT95GbQTByvYOo639Nfq4LBEVyvhYdoc,2289
-yirgacheffe/window.py,sha256=QuyBLOwKFI0XkEQ4Bd2hdELPbJSfHL7mt5KSi7CIHcE,9505
-yirgacheffe/_backends/__init__.py,sha256=jN-2iRrHStnPI6cNL7XhwhsROtI0EaGfIrbF5c-ECV0,334
-yirgacheffe/_backends/enumeration.py,sha256=9bcCXz9Ssrh8Oh1iazodkx6Gm2kQBi9HQ9z9zehS4AE,1806
-yirgacheffe/_backends/mlx.py,sha256=U1gl1lK1mZXLEET6ylF1TNs6WJ0PBEvfSk7ppn28n8w,6203
-yirgacheffe/_backends/numpy.py,sha256=Gxx49JJH79GFEkKIpV6IyjCUcdtN5-qLlzRfylzKhS4,4142
-yirgacheffe/layers/__init__.py,sha256=mYKjw5YTcMNv_hMy7a6K4yRzIuNUbR8WuBTw4WIAmSk,435
-yirgacheffe/layers/area.py,sha256=wJcMHbLJBaXS4BeFbu5rYeKfgu3gvaE9hwQ5j6aw-y4,3976
-yirgacheffe/layers/base.py,sha256=7b4WXuvnmCv8mR0iyCIuSEolnV8D3f2vtCaYlcJCIa8,13201
-yirgacheffe/layers/constant.py,sha256=gtkQ98Z01CYYDgFElswtRZY4ZG3UnS5NIAoIVue5ufk,1481
-yirgacheffe/layers/group.py,sha256=yaqf-ra_Vh59yrWcz7-OvJ1fBnTcBXZd18AfRDN5Ymo,16157
-yirgacheffe/layers/h3layer.py,sha256=Rq1bFo7CApIh5NdBcV7hSj3hm-DszY79nhYsTRAvJ_g,9916
-yirgacheffe/layers/rasters.py,sha256=zBE9uXm6LvAQF2_XdQzcOgJQOQWGmuPflY5JNDrUf3k,13527
-yirgacheffe/layers/rescaled.py,sha256=gEFbXeYxX1nVn7eQYmbGww90_yc5ENmgQrD_WxXxpQE,3352
-yirgacheffe/layers/vectors.py,sha256=A27kuTr0C9BZhHG0-cplNEa7aSNcse37Pm9xTjEzv-c,19990
-yirgacheffe-1.9.4.dist-info/licenses/LICENSE,sha256=dNSHwUCJr6axStTKDEdnJtfmDdFqlE3h1NPCveqPfnY,757
-yirgacheffe-1.9.4.dist-info/METADATA,sha256=cUAABIwPG9_RTO0ylH25O_7usbtQneZwTUCsfk_SLIs,7407
-yirgacheffe-1.9.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-yirgacheffe-1.9.4.dist-info/entry_points.txt,sha256=j4KgHXbVGbGyfTySc1ypBdERpfihO4WNjppvCdE9HjE,52
-yirgacheffe-1.9.4.dist-info/top_level.txt,sha256=9DBFlKO2Ld3hG6TuE3qOTd3Tt8ugTiXil4AN4Wr9_y0,12
-yirgacheffe-1.9.4.dist-info/RECORD,,