yirgacheffe 1.10.2__tar.gz → 1.10.3__tar.gz

{yirgacheffe-1.10.2/yirgacheffe.egg-info → yirgacheffe-1.10.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.10.2
+Version: 1.10.3
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC

{yirgacheffe-1.10.2 → yirgacheffe-1.10.3}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "yirgacheffe"
-version = "1.10.2"
+version = "1.10.3"
 description = "Abstraction of gdal datasets for doing basic math operations"
 readme = "README.md"
 authors = [{ name = "Michael Dales", email = "mwd24@cam.ac.uk" }]

yirgacheffe-1.10.3/tests/test_aggregations.py

@@ -0,0 +1,68 @@
+import json
+
+import numpy as np
+import pytest
+
+import yirgacheffe as yg
+from yirgacheffe._backends import backend, BACKEND
+
+def test_sum_result_is_scalar() -> None:
+    rng = np.random.default_rng(seed=42)
+    data = rng.integers(0, 128, size=(1000, 1000))
+    with yg.from_array(data, (0, 0), ("epsg:4326", (0.01, -0.01))) as layer:
+        total = layer.sum()
+        json.dumps(total)
+
+@pytest.mark.parametrize("c,dtype,maxval", [
+    (int(2), np.int8, 120),
+    (int(2), np.uint8, 250),
+    (int(2), np.int16, 32000),
+    (int(2), np.uint16, 64000),
+    (int(2), np.int32, 66000),
+    (int(2), np.uint32, 66000),
+])
+@pytest.mark.parametrize("step", [1, 2, 4, 8])
+def test_sums_of_calc_int(monkeypatch, step, c, dtype: type, maxval: int) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yg.constants, "YSTEP", step)
+
+        rng = np.random.default_rng(seed=42)
+
+        data = rng.integers(0, maxval, size=(1000, 1000), dtype=dtype)
+        typed_data = backend.promote(data)
+
+        assert np.sum(data) == backend.sum_op(typed_data)
+
+        with yg.from_array(data, (0, 0), ("epsg:4326", (0.01, -0.01))) as layer:
+            assert layer.sum() == backend.sum_op(typed_data)
+            calc = layer * c
+            actual = calc.sum()
+            expected = backend.sum_op(typed_data * c)
+            assert actual == expected
+
+@pytest.mark.parametrize("c,dtype,maxval", [
+    (float(2.5), np.int8, 120),
+    (float(2.5), np.uint8, 250),
+    (float(2.5), np.int16, 32000),
+    (float(2.5), np.uint16, 640),
+    (float(2.5), np.int32, 660),
+    (float(2.5), np.uint32, 660),
+])
+@pytest.mark.parametrize("step", [1, 2, 4, 8])
+def test_sums_of_calc_float_mlx(monkeypatch, step, c, dtype: type, maxval: int) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yg.constants, "YSTEP", step)
+
+        rng = np.random.default_rng(seed=42)
+
+        data = rng.integers(0, maxval, size=(100, 100), dtype=dtype)
+        typed_data = backend.promote(data)
+
+        with yg.from_array(data, (0, 0), ("epsg:4326", (0.01, -0.01))) as layer:
+            assert layer.sum() == backend.sum_op(typed_data)
+            calc = layer * c
+            actual = calc.sum()
+            expected = backend.sum_op(typed_data * c)
+            # MLX has a maximum float size of 32 bit for GPU and NUMPY has 64 bit on CPU
+            rel = 1e-6 if BACKEND == "MLX" else 1e-10
+            assert float(actual) == pytest.approx(float(expected), rel=rel)

{yirgacheffe-1.10.2 → yirgacheffe-1.10.3}/tests/test_constants.py

@@ -23,9 +23,9 @@ def test_constant_parallel_save(monkeypatch) -> None:
     area = Area(left=-1.0, right=1.0, top=1.0, bottom=-1.0)
     scale = PixelScale(0.1, -0.1)
     with RasterLayer.empty_raster_layer(area, scale, DataType.Float32) as result:
-        with ConstantLayer(42.0) as c:
-            with monkeypatch.context() as m:
-                m.setattr(yg.constants, "YSTEP", 1)
+        with monkeypatch.context() as m:
+            m.setattr(yg.constants, "YSTEP", 1)
+            with ConstantLayer(42.0) as c:
                 c.parallel_save(result)
 
         expected = np.full((20, 20), 42.0)

{yirgacheffe-1.10.2 → yirgacheffe-1.10.3}/tests/test_pickle.py

@@ -134,8 +134,6 @@ def test_pickle_simple_calc(c) -> None:
         layer = RasterLayer(gdal_dataset_of_region(area, 0.2, filename=path))
 
         calc = layer * c
-        assert calc.sum() != 0
-        assert calc.sum() == layer.sum() * c
 
         p = pickle.dumps(calc)
         restore = pickle.loads(p)

{yirgacheffe-1.10.2 → yirgacheffe-1.10.3}/yirgacheffe/_backends/mlx.py

@@ -14,7 +14,7 @@ float_t = mx.float32
 
 promote = mx.array
 demote_array = np.asarray
-demote_scalar = np.float64
+demote_scalar = lambda a: a.item()
 
 eval_op = mx.eval
 
@@ -54,15 +54,25 @@ round_op = mx.round
 ceil_op = mx.ceil
 
 def sum_op(a):
-    # There are weird issues around how MLX overflows int8, so just promote the data ahead of summing
+    # By default the type promotion rules for sum in MLX are not the same as with Numpy. E.g.,
+    #
+    # >>> mx.array(np.array([1, 2, 3], dtype=np.uint8)).sum()
+    # array(6, dtype=uint32)
+    # >>> np.array([1, 2, 3], dtype=np.uint8).sum()
+    # np.uint64(6)
+    #
+    # This has problems for Yirgacheffe as it means we get different answers depending on which
+    # backend is active. MLX is more conservative, which also can cause issues with Geospatial
+    # datasets being large and causing overflows. Thus we force promotion here to match what
+    # numpy does, even if this means things run a little slower.
     match a.dtype:
-        case mx.int8:
-            res = mx.sum(a.astype(mx.int32))
-        case mx.uint8:
-            res = mx.sum(a.astype(mx.uint32))
+        case mx.int8 | mx.int16 | mx.int32:
+            res = mx.sum(a.astype(mx.int64))
+        case mx.uint8 | mx.uint16 | mx.uint32:
+            res = mx.sum(a.astype(mx.uint64))
         case _:
             res = mx.sum(a)
-    return demote_scalar(res)
+    return res
 
 def _is_float(x):
     if isinstance(x, float):

{yirgacheffe-1.10.2 → yirgacheffe-1.10.3}/yirgacheffe/_operators/__init__.py

@@ -774,11 +774,11 @@ class LayerOperation(LayerMathMixin):
         return result
 
     def sum(self) -> float:
-        # The result accumulator is float64, and for precision reasons
-        # we force the sum to be done in float64 also. Otherwise we
-        # see variable results depending on chunk size, as different parts
-        # of the sum are done in different types.
-        res = 0.0
+        # Numpy and MLX have different behaviours that make guessing
+        # the type of the accumulator tricky, so we go by the type of the
+        # first chunk returned
+        res = None
+
         computation_window = self.window
         projection = self.map_projection
         if projection is None:
@@ -799,8 +799,9 @@ class LayerOperation(LayerMathMixin):
                 step,
                 computation_window
             )
-            res += backend.sum_op(chunk)
-        return res
+            chunk_sum = backend.sum_op(chunk)
+            res = chunk_sum if res is None else res + chunk_sum
+        return backend.demote_scalar(res) if res is not None else 0
 
     def min(self) -> float:
         res = float('inf')
@@ -894,7 +895,7 @@ class LayerOperation(LayerMathMixin):
                 f"{(destination_window.xsize, destination_window.ysize)} vs "
                 f"{(computation_window.xsize, computation_window.ysize)}"))
 
-        total = 0.0
+        total = None
 
         cse_cache = CSECacheTable(self, computation_window)
 
@@ -916,11 +917,12 @@ class LayerOperation(LayerMathMixin):
                 yoffset + destination_window.yoff,
             )
             if and_sum:
-                total += backend.sum_op(chunk)
+                chunk_sum = backend.sum_op(chunk)
+                total = chunk_sum if total is None else total + chunk_sum
         if callback:
             callback(1.0)
 
-        return total if and_sum else None
+        return backend.demote_scalar(total) if and_sum else None
 
     def _parallel_worker(
         self,
@@ -1030,7 +1032,7 @@ class LayerOperation(LayerMathMixin):
 
         if destination_layer is not None:
             try:
-                band = destination_layer._dataset.GetRasterBand(band)
+                gdal_band = destination_layer._dataset.GetRasterBand(band)
             except AttributeError as exc:
                 raise ValueError("Layer must be a raster backed layer") from exc
 
@@ -1052,16 +1054,23 @@ class LayerOperation(LayerMathMixin):
                 gdal.GDT_UInt32:  np.dtype('uint32'),
                 gdal.GDT_UInt64:  np.dtype('uint64'),
             }
-            np_dtype = np_type_map[band.DataType]
+            np_dtype = np_type_map[gdal_band.DataType]
         else:
-            band = None
-            np_dtype = np.dtype('float64')
+            # the aggregation path
+            gdal_band = None
+            match self.datatype:
+                case DataType.Int8 | DataType.Int16 | DataType.Int32 | DataType.Int64:
+                    np_dtype = np.dtype('int64')
+                case DataType.UInt8 | DataType.UInt16 | DataType.UInt32 | DataType.UInt64:
+                    np_dtype = np.dtype('uint64')
+                case _:
+                    np_dtype = np.dtype('float64')
 
         # The parallel save will cause a fork on linux, so we need to
         # remove all SWIG references
         self._park()
 
-        total = 0.0
+        total = None
 
         with ExitStack() as stack:
             # If we get this far, then we're going to do the multiprocessing path. In general we've had
@@ -1125,14 +1134,15 @@ class LayerOperation(LayerMathMixin):
                             continue
                         index, yoffset, step = res
                         _, sem, arr = mem_sem_cast[index]
-                        if band:
-                            band.WriteArray(
+                        if gdal_band:
+                            gdal_band.WriteArray(
                                 arr[0:step],
                                 destination_window.xoff,
                                 yoffset + destination_window.yoff,
                             )
                         if and_sum:
-                            total += np.sum(np.array(arr[0:step]).astype(np.float64))
+                            chunk_sum = np.sum(np.array(arr[0:step]).astype(np.float64))
+                            total = chunk_sum if total is None else total + chunk_sum
                         sem.release()
                         retired_blocks += 1
                         if callback:
@@ -1150,7 +1160,7 @@ class LayerOperation(LayerMathMixin):
                             processes.remove(candidate)
                         time.sleep(0.01)
 
-        return total if and_sum else None
+        return backend.demote_scalar(total) if and_sum else None
 
     def parallel_save(
         self,

{yirgacheffe-1.10.2 → yirgacheffe-1.10.3}/yirgacheffe/layers/area.py

@@ -9,6 +9,7 @@ from osgeo import gdal
 
 from ..window import Area, Window
 from .rasters import RasterLayer
+from .._backends import backend
 
 class UniformAreaLayer(RasterLayer):
     """If you have a pixel area map where all the row entries are identical, then you
@@ -98,4 +99,4 @@ class UniformAreaLayer(RasterLayer):
         if ysize <= 0:
             raise ValueError("Request dimensions must be positive and non-zero")
         offset = window.yoff + yoffset
-        return self.databand[offset:offset + ysize]
+        return backend.promote(self.databand[offset:offset + ysize])

{yirgacheffe-1.10.2 → yirgacheffe-1.10.3/yirgacheffe.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.10.2
+Version: 1.10.3
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC

{yirgacheffe-1.10.2 → yirgacheffe-1.10.3}/yirgacheffe.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ LICENSE
 MANIFEST.in
 README.md
 pyproject.toml
+tests/test_aggregations.py
 tests/test_area.py
 tests/test_auto_windowing.py
 tests/test_constants.py