yirgacheffe 1.3.1__tar.gz → 1.3.3__tar.gz

{yirgacheffe-1.3.1/yirgacheffe.egg-info → yirgacheffe-1.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.3.1
+Version: 1.3.3
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC
@@ -304,9 +304,11 @@ with RasterLayer.layer_from_file('test1.tif') as layer1:
 The following math operators common to numpy and other libraries are currently supported:
 
 * abs
+* ceil
 * clip
 * exp
 * exp2
+* floor
 * isin
 * log
 * log2
@@ -314,6 +316,7 @@ The following math operators common to numpy and other libraries are currently s
 * maximum
 * minimum
 * nan_to_num
+* round
 
 Typically these can be invoked either on a layer as a method:
 

{yirgacheffe-1.3.1 → yirgacheffe-1.3.3}/README.md

@@ -280,9 +280,11 @@ with RasterLayer.layer_from_file('test1.tif') as layer1:
 The following math operators common to numpy and other libraries are currently supported:
 
 * abs
+* ceil
 * clip
 * exp
 * exp2
+* floor
 * isin
 * log
 * log2
@@ -290,6 +292,7 @@ The following math operators common to numpy and other libraries are currently s
 * maximum
 * minimum
 * nan_to_num
+* round
 
 Typically these can be invoked either on a layer as a method:
 

{yirgacheffe-1.3.1 → yirgacheffe-1.3.3}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "yirgacheffe"
-version = "1.3.1"
+version = "1.3.3"
 description = "Abstraction of gdal datasets for doing basic math operations"
 readme = "README.md"
 authors = [{ name = "Michael Dales", email = "mwd24@cam.ac.uk" }]

{yirgacheffe-1.3.1 → yirgacheffe-1.3.3}/tests/test_operators.py

@@ -1332,3 +1332,87 @@ def test_simple_conv2d_over_calculated_result(skip) -> None:
 
                 # Torch and MLX give slightly different rounding
                 assert np.isclose(expected, actual).all()
+
+def test_floor_method() -> None:
+    data1 = np.array([[1.0, 1.2, 1.5, 1.7], [-1.0, -1.2, -1.5, -1.7]])
+    layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
+    result = RasterLayer.empty_raster_layer_like(layer1)
+
+    comp = layer1.floor()
+    comp.save(result)
+
+    expected = backend.floor_op(backend.promote(data1))
+    backend.eval_op(expected)
+
+    actual = result.read_array(0, 0, 4, 2)
+    assert (expected == actual).all()
+
+def test_floor_module() -> None:
+    data1 = np.array([[1.0, 1.2, 1.5, 1.7], [-1.0, -1.2, -1.5, -1.7]])
+    layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
+    result = RasterLayer.empty_raster_layer_like(layer1)
+
+    comp = LayerOperation.floor(layer1)
+    comp.save(result)
+
+    expected = backend.floor_op(backend.promote(data1))
+    backend.eval_op(expected)
+
+    actual = result.read_array(0, 0, 4, 2)
+    assert (expected == actual).all()
+
+def test_round_method() -> None:
+    data1 = np.array([[1.0, 1.2, 1.5, 1.7], [-1.0, -1.2, -1.5, -1.7]])
+    layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
+    result = RasterLayer.empty_raster_layer_like(layer1)
+
+    comp = layer1.round()
+    comp.save(result)
+
+    expected = backend.round_op(backend.promote(data1))
+    backend.eval_op(expected)
+
+    actual = result.read_array(0, 0, 4, 2)
+    assert (expected == actual).all()
+
+def test_round_module() -> None:
+    data1 = np.array([[1.0, 1.2, 1.5, 1.7], [-1.0, -1.2, -1.5, -1.7]])
+    layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
+    result = RasterLayer.empty_raster_layer_like(layer1)
+
+    comp = LayerOperation.round(layer1)
+    comp.save(result)
+
+    expected = backend.round_op(backend.promote(data1))
+    backend.eval_op(expected)
+
+    actual = result.read_array(0, 0, 4, 2)
+    assert (expected == actual).all()
+
+def test_ceil_method() -> None:
+    data1 = np.array([[1.0, 1.2, 1.5, 1.7], [-1.0, -1.2, -1.5, -1.7]])
+    layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
+    result = RasterLayer.empty_raster_layer_like(layer1)
+
+    comp = layer1.ceil()
+    comp.save(result)
+
+    expected = backend.ceil_op(backend.promote(data1))
+    backend.eval_op(expected)
+
+    actual = result.read_array(0, 0, 4, 2)
+    assert (expected == actual).all()
+
+def test_ceil_module() -> None:
+    data1 = np.array([[1.0, 1.2, 1.5, 1.7], [-1.0, -1.2, -1.5, -1.7]])
+    layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
+    result = RasterLayer.empty_raster_layer_like(layer1)
+
+    comp = LayerOperation.ceil(layer1)
+    comp.save(result)
+
+    expected = backend.ceil_op(backend.promote(data1))
+    backend.eval_op(expected)
+
+    actual = result.read_array(0, 0, 4, 2)
+    assert (expected == actual).all()

yirgacheffe-1.3.3/tests/test_parallel_operators.py

@@ -0,0 +1,310 @@
+import os
+import tempfile
+
+import numpy as np
+import pytest
+import torch
+
+import yirgacheffe
+from helpers import gdal_dataset_with_data
+from yirgacheffe.layers import RasterLayer, ConstantLayer
+from yirgacheffe.operators import LayerOperation
+
+# These tests are marked skip for MLX, because there seems to be a problem with
+# calling mx.eval in the tests for parallel save on Linux (which is what we use
+# for github actions for instance). They do pass on macOS - but that could also
+# just be down to Python versions. To add further complication, if I just run
+# this file along on linux with MLX it passes fine 🤦
+#
+# It seems that under the hood MLX is doing some threading of its own and my
+# guess is that that's interacting with the Python threading here.
+
+def test_add_byte_layers_with_one_thread_uses_regular_save(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 4)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+
+            path2 = os.path.join(tempdir, "test2.tif")
+            data2 = np.array([[10, 20, 30, 40], [50, 60, 70, 80]])
+            dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2, filename=path2)
+            dataset2.Close()
+
+            layer1 = RasterLayer.layer_from_file(path1)
+            layer2 = RasterLayer.layer_from_file(path2)
+            result = RasterLayer.empty_raster_layer_like(layer1)
+
+            comp = layer1 + layer2
+            with pytest.raises(TypeError):
+                comp.parallel_save(result)
+
+
+@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_add_byte_layers(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+
+            path2 = os.path.join(tempdir, "test2.tif")
+            data2 = np.array([[10, 20, 30, 40], [50, 60, 70, 80]])
+            dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2, filename=path2)
+            dataset2.Close()
+
+            layer1 = RasterLayer.layer_from_file(path1)
+            layer2 = RasterLayer.layer_from_file(path2)
+            result = RasterLayer.empty_raster_layer_like(layer1)
+
+            comp = layer1 + layer2
+            comp.parallel_save(result)
+
+            expected = data1 + data2
+            actual = result.read_array(0, 0, 4, 2)
+
+            assert (expected == actual).all()
+
+@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_add_byte_layers_and_sum(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+
+            path2 = os.path.join(tempdir, "test2.tif")
+            data2 = np.array([[10, 20, 30, 40], [50, 60, 70, 80]])
+            dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2, filename=path2)
+            dataset2.Close()
+
+            layer1 = RasterLayer.layer_from_file(path1)
+            layer2 = RasterLayer.layer_from_file(path2)
+            result = RasterLayer.empty_raster_layer_like(layer1)
+
+            comp = layer1 + layer2
+            sum_total = comp.parallel_save(result, and_sum=True)
+
+            expected = data1 + data2
+            actual = result.read_array(0, 0, 4, 2)
+
+            assert (expected == actual).all()
+            assert sum_total == expected.sum()
+
+@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_parallel_sum(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+
+            path2 = os.path.join(tempdir, "test2.tif")
+            data2 = np.array([[10, 20, 30, 40], [50, 60, 70, 80]])
+            dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2, filename=path2)
+            dataset2.Close()
+
+            layer1 = RasterLayer.layer_from_file(path1)
+            layer2 = RasterLayer.layer_from_file(path2)
+
+            comp = layer1 + layer2
+            sum_total = comp.parallel_sum()
+
+            expected = data1 + data2
+            assert sum_total == expected.sum()
+
+@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+@pytest.mark.parametrize("skip,expected_steps", [
+    (1, [0.0, 0.25, 0.5, 0.75, 1.0]),
+    (2, [0.0, 0.5, 1.0]),
+])
+def test_parallel_with_different_skip(monkeypatch, skip, expected_steps) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8], [1, 2, 3, 4], [5, 6, 7, 8]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+
+            path2 = os.path.join(tempdir, "test2.tif")
+            data2 = np.array([[10, 20, 30, 40], [50, 60, 70, 80], [10, 20, 30, 40], [50, 60, 70, 80]])
+            dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2, filename=path2)
+            dataset2.Close()
+
+            layer1 = RasterLayer.layer_from_file(path1)
+            layer2 = RasterLayer.layer_from_file(path2)
+            result = RasterLayer.empty_raster_layer_like(layer1)
+
+            callback_possitions = []
+
+            comp = layer1 + layer2
+            comp.ystep = skip
+            comp.parallel_save(result, callback=lambda x: callback_possitions.append(x))
+
+            expected = data1 + data2
+            actual = result.read_array(0, 0, 4, 4)
+
+            assert (expected == actual).all()
+
+            assert callback_possitions == expected_steps
+
+@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_parallel_equality(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[1, 2, 3, 4], [4, 3, 2, 1]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+            with RasterLayer.layer_from_file(path1) as layer1:
+                with RasterLayer.empty_raster_layer_like(layer1) as result:
+                    comp = layer1 == 2
+                    comp.parallel_save(result)
+
+                    expected = np.array([[0, 1, 0, 0], [0, 0, 1, 0]])
+                    actual = result.read_array(0, 0, 4, 2)
+
+                    assert (expected == actual).all()
+
+@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_parallel_equality_to_file(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            path2 = os.path.join(tempdir, "result.tif")
+            data1 = np.array([[1, 2, 3, 4], [4, 3, 2, 1]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+            with RasterLayer.layer_from_file(path1) as layer1:
+                comp = layer1 == 2
+                with RasterLayer.empty_raster_layer_like(layer1, filename=path2) as result:
+                    comp.parallel_save(result)
+            with RasterLayer.layer_from_file(path2) as actual_result:
+                expected = np.array([[0, 1, 0, 0], [0, 0, 1, 0]])
+                actual = actual_result.read_array(0, 0, 4, 2)
+                assert (expected == actual).all()
+
+@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_parallel_unary_numpy_apply_with_function(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+            layer1 = RasterLayer.layer_from_file(path1)
+
+            result = RasterLayer.empty_raster_layer_like(layer1)
+
+            def simple_add(chunk):
+                return chunk + 1.0
+
+            comp = layer1.numpy_apply(simple_add)
+            comp.ystep = 1
+            comp.parallel_save(result)
+
+            expected = data1 + 1.0
+            actual = result.read_array(0, 0, 4, 2)
+
+            assert (expected == actual).all()
+
+@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_parallel_unary_numpy_apply_with_lambda(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+            layer1 = RasterLayer.layer_from_file(path1)
+
+            result = RasterLayer.empty_raster_layer_like(layer1)
+
+            comp = layer1.numpy_apply(lambda a: a + 1.0)
+            comp.ystep = 1
+            comp.parallel_save(result)
+
+            expected = data1 + 1.0
+            actual = result.read_array(0, 0, 4, 2)
+
+            assert (expected == actual).all()
+
+@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_parallel_where_simple(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[0, 1, 0, 2], [0, 0, 1, 1]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+            layer1 = RasterLayer.layer_from_file(path1)
+
+            result = RasterLayer.empty_raster_layer_like(layer1)
+
+            comp = LayerOperation.where(layer1 > 0, 1, 2)
+            comp.ystep = 1
+            comp.parallel_save(result)
+
+            expected = np.where(data1 > 0, 1, 2)
+            actual = result.read_array(0, 0, 4, 2)
+            assert (expected == actual).all()
+
+@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_parallel_conv2d() -> None:
+    with tempfile.TemporaryDirectory() as tempdir:
+
+        data1 = np.array([
+            [0, 0, 0, 0, 0],
+            [0, 1, 1, 1, 0],
+            [0, 1, 1, 1, 0],
+            [0, 1, 1, 1, 0],
+            [0, 0, 0, 0, 0],
+        ]).astype(np.float32)
+        weights = np.array([
+            [0.0, 0.1, 0.0],
+            [0.1, 0.6, 0.1],
+            [0.0, 0.1, 0.0],
+        ])
+
+        conv = torch.nn.Conv2d(1, 1, 3, padding=1, bias=False)
+        conv.weight = torch.nn.Parameter(torch.from_numpy(np.array([[weights.astype(np.float32)]])))
+        tensorres = conv(torch.from_numpy(np.array([[data1]])))
+        expected = tensorres.detach().numpy()[0][0]
+
+        path1 = os.path.join(tempdir, "test1.tif")
+        dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+        dataset1.Close()
+
+        with RasterLayer.layer_from_file(path1) as layer1:
+
+            calc = layer1.conv2d(weights)
+            with RasterLayer.empty_raster_layer_like(layer1) as res:
+                calc.save(res)
+                actual = res.read_array(0, 0, 5, 5)
+
+                # Torch and MLX give slightly different rounding
+                assert np.isclose(expected, actual).all()

{yirgacheffe-1.3.1 → yirgacheffe-1.3.3}/yirgacheffe/backends/enumeration.py

@@ -35,7 +35,9 @@ class operators(Enum):
     CONV2D = 30
     ABS = 31
     ASTYPE = 32
-
+    FLOOR = 33
+    ROUND = 34
+    CEIL = 35
 
 class dtype(Enum):
     Float32 = gdal.GDT_Float32

{yirgacheffe-1.3.1 → yirgacheffe-1.3.3}/yirgacheffe/backends/mlx.py

@@ -45,6 +45,9 @@ allclose = mx.allclose
 remainder_op = mx.remainder
 floordiv_op = mx.array.__floordiv__
 abs_op = mx.abs
+floor_op = mx.floor
+round_op = mx.round
+ceil_op = mx.ceil
 
 def sum_op(a):
     # There are weird issues around how MLX overflows int8, so just promote the data ahead of summing
@@ -211,4 +214,7 @@ operator_map = {
     op.CONV2D: conv2d_op,
     op.ABS: mx.abs,
     op.ASTYPE: astype_op,
+    op.FLOOR: mx.floor,
+    op.ROUND: mx.round,
+    op.CEIL: mx.ceil,
 }

{yirgacheffe-1.3.1 → yirgacheffe-1.3.3}/yirgacheffe/backends/numpy.py

@@ -48,6 +48,9 @@ allclose = np.allclose
 remainder_op = np.ndarray.__mod__
 floordiv_op = np.ndarray.__floordiv__
 abs_op = np.abs
+floor_op = np.floor
+round_op = np.round
+ceil_op = np.ceil
 
 def conv2d_op(data, weights):
     # torch wants to process dimensions of channels of width of height
@@ -149,4 +152,7 @@ operator_map = {
     op.CONV2D: conv2d_op,
     op.ABS: np.abs,
     op.ASTYPE: astype_op,
+    op.FLOOR: np.floor,
+    op.ROUND: np.round,
+    op.CEIL: np.ceil,
 }

yirgacheffe-1.3.3/yirgacheffe/constants.py

@@ -0,0 +1,2 @@
+YSTEP = 512
+MINIMUM_CHUNKS_PER_THREAD = 1

{yirgacheffe-1.3.1 → yirgacheffe-1.3.3}/yirgacheffe/operators.py

@@ -1,4 +1,5 @@
 import logging
+import math
 import multiprocessing
 import sys
 import time
@@ -120,6 +121,27 @@ class LayerMathMixin:
             window_op=WindowOperation.NONE,
         )
 
+    def floor(self):
+        return LayerOperation(
+            self,
+            op.FLOOR,
+            window_op=WindowOperation.NONE,
+        )
+
+    def round(self):
+        return LayerOperation(
+            self,
+            op.ROUND,
+            window_op=WindowOperation.NONE,
+        )
+
+    def ceil(self):
+        return LayerOperation(
+            self,
+            op.CEIL,
+            window_op=WindowOperation.NONE,
+        )
+
     def log(self):
         return LayerOperation(
             self,
@@ -578,6 +600,20 @@ class LayerOperation(LayerMathMixin):
     def _parallel_save(self, destination_layer, and_sum=False, callback=None, parallelism=None, band=1):
         assert (destination_layer is not None) or and_sum
         computation_window = self.window
+
+        worker_count = parallelism or multiprocessing.cpu_count()
+        work_blocks = len(range(0, computation_window.ysize, self.ystep))
+        adjusted_blocks = math.ceil(work_blocks / constants.MINIMUM_CHUNKS_PER_THREAD)
+        worker_count = min(adjusted_blocks, worker_count)
+
+        if worker_count == 1:
+            if destination_layer:
+                return self.save(destination_layer, and_sum, callback, band)
+            elif and_sum:
+                return self.sum()
+            else:
+                assert False
+
         if destination_layer is not None:
             try:
                 band = destination_layer._dataset.GetRasterBand(band)
@@ -615,10 +651,6 @@ class LayerOperation(LayerMathMixin):
         with multiprocessing.Manager() as manager:
             with SharedMemoryManager() as smm:
 
-                worker_count = parallelism or multiprocessing.cpu_count()
-                work_blocks = len(range(0, computation_window.ysize, self.ystep))
-                worker_count = min(work_blocks, worker_count)
-
                 mem_sem_cast = []
                 for i in range(worker_count):
                     shared_buf = smm.SharedMemory(size=np_dtype.itemsize * self.ystep * computation_window.xsize)
@@ -754,3 +786,6 @@ exp2 = LayerOperation.exp2
 nan_to_num = LayerOperation.nan_to_num
 isin = LayerOperation.isin
 abs = LayerOperation.abs # pylint: disable=W0622
+floor = LayerOperation.floor
+round = LayerOperation.round # pylint: disable=W0622
+ceil = LayerOperation.ceil

{yirgacheffe-1.3.1 → yirgacheffe-1.3.3/yirgacheffe.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.3.1
+Version: 1.3.3
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC
@@ -304,9 +304,11 @@ with RasterLayer.layer_from_file('test1.tif') as layer1:
 The following math operators common to numpy and other libraries are currently supported:
 
 * abs
+* ceil
 * clip
 * exp
 * exp2
+* floor
 * isin
 * log
 * log2
@@ -314,6 +316,7 @@ The following math operators common to numpy and other libraries are currently s
 * maximum
 * minimum
 * nan_to_num
+* round
 
 Typically these can be invoked either on a layer as a method:
 

yirgacheffe-1.3.1/tests/test_parallel_operators.py

@@ -1,255 +0,0 @@
-import os
-import tempfile
-
-import numpy as np
-import pytest
-import torch
-
-import yirgacheffe
-from helpers import gdal_dataset_with_data
-from yirgacheffe.layers import RasterLayer, ConstantLayer
-from yirgacheffe.operators import LayerOperation
-
-# These tests are marked skip for MLX, because there seems to be a problem with
-# calling mx.eval in the tests for parallel save on Linux (which is what we use
-# for github actions for instance). They do pass on macOS - but that could also
-# just be down to Python versions. To add further complication, if I just run
-# this file along on linux with MLX it passes fine 🤦
-#
-# It seems that under the hood MLX is doing some threading of its own and my
-# guess is that that's interacting with the Python threading here.
-
-
-@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
-def test_add_byte_layers() -> None:
-    with tempfile.TemporaryDirectory() as tempdir:
-        path1 = os.path.join(tempdir, "test1.tif")
-        data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
-        dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
-        dataset1.Close()
-
-        path2 = os.path.join(tempdir, "test2.tif")
-        data2 = np.array([[10, 20, 30, 40], [50, 60, 70, 80]])
-        dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2, filename=path2)
-        dataset2.Close()
-
-        layer1 = RasterLayer.layer_from_file(path1)
-        layer2 = RasterLayer.layer_from_file(path2)
-        result = RasterLayer.empty_raster_layer_like(layer1)
-
-        comp = layer1 + layer2
-        comp.parallel_save(result)
-
-        expected = data1 + data2
-        actual = result.read_array(0, 0, 4, 2)
-
-        assert (expected == actual).all()
-
-@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
-def test_add_byte_layers_and_sum() -> None:
-    with tempfile.TemporaryDirectory() as tempdir:
-        path1 = os.path.join(tempdir, "test1.tif")
-        data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
-        dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
-        dataset1.Close()
-
-        path2 = os.path.join(tempdir, "test2.tif")
-        data2 = np.array([[10, 20, 30, 40], [50, 60, 70, 80]])
-        dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2, filename=path2)
-        dataset2.Close()
-
-        layer1 = RasterLayer.layer_from_file(path1)
-        layer2 = RasterLayer.layer_from_file(path2)
-        result = RasterLayer.empty_raster_layer_like(layer1)
-
-        comp = layer1 + layer2
-        sum_total = comp.parallel_save(result, and_sum=True)
-
-        expected = data1 + data2
-        actual = result.read_array(0, 0, 4, 2)
-
-        assert (expected == actual).all()
-        assert sum_total == expected.sum()
-
-@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
-def test_parallel_sum() -> None:
-    with tempfile.TemporaryDirectory() as tempdir:
-        path1 = os.path.join(tempdir, "test1.tif")
-        data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
-        dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
-        dataset1.Close()
-
-        path2 = os.path.join(tempdir, "test2.tif")
-        data2 = np.array([[10, 20, 30, 40], [50, 60, 70, 80]])
-        dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2, filename=path2)
-        dataset2.Close()
-
-        layer1 = RasterLayer.layer_from_file(path1)
-        layer2 = RasterLayer.layer_from_file(path2)
-
-        comp = layer1 + layer2
-        sum_total = comp.parallel_sum()
-
-        expected = data1 + data2
-        assert sum_total == expected.sum()
-
-@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
-@pytest.mark.parametrize("skip,expected_steps", [
-    (1, [0.0, 0.5, 1.0]),
-    (2, [0.0, 1.0]),
-])
-def test_parallel_with_different_skip(skip, expected_steps) -> None:
-    with tempfile.TemporaryDirectory() as tempdir:
-        path1 = os.path.join(tempdir, "test1.tif")
-        data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
-        dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
-        dataset1.Close()
-
-        path2 = os.path.join(tempdir, "test2.tif")
-        data2 = np.array([[10, 20, 30, 40], [50, 60, 70, 80]])
-        dataset2 = gdal_dataset_with_data((0.0, 0.0), 0.02, data2, filename=path2)
-        dataset2.Close()
-
-        layer1 = RasterLayer.layer_from_file(path1)
-        layer2 = RasterLayer.layer_from_file(path2)
-        result = RasterLayer.empty_raster_layer_like(layer1)
-
-        callback_possitions = []
-
-        comp = layer1 + layer2
-        comp.ystep = skip
-        comp.parallel_save(result, callback=lambda x: callback_possitions.append(x))
-
-        expected = data1 + data2
-        actual = result.read_array(0, 0, 4, 2)
-
-        assert (expected == actual).all()
-
-        assert callback_possitions == expected_steps
-
-@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
-def test_parallel_equality() -> None:
-    with tempfile.TemporaryDirectory() as tempdir:
-        path1 = os.path.join(tempdir, "test1.tif")
-        data1 = np.array([[1, 2, 3, 4], [4, 3, 2, 1]])
-        dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
-        dataset1.Close()
-        with RasterLayer.layer_from_file(path1) as layer1:
-            with RasterLayer.empty_raster_layer_like(layer1) as result:
-                comp = layer1 == 2
-                comp.parallel_save(result)
-
-                expected = np.array([[0, 1, 0, 0], [0, 0, 1, 0]])
-                actual = result.read_array(0, 0, 4, 2)
-
-                assert (expected == actual).all()
-
-@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
-def test_parallel_equality_to_file() -> None:
-    with tempfile.TemporaryDirectory() as tempdir:
-        path1 = os.path.join(tempdir, "test1.tif")
-        path2 = os.path.join(tempdir, "result.tif")
-        data1 = np.array([[1, 2, 3, 4], [4, 3, 2, 1]])
-        dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
-        dataset1.Close()
-        with RasterLayer.layer_from_file(path1) as layer1:
-            comp = layer1 == 2
-            with RasterLayer.empty_raster_layer_like(layer1, filename=path2) as result:
-                comp.parallel_save(result)
-        with RasterLayer.layer_from_file(path2) as actual_result:
-            expected = np.array([[0, 1, 0, 0], [0, 0, 1, 0]])
-            actual = actual_result.read_array(0, 0, 4, 2)
-            assert (expected == actual).all()
-
-@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
-def test_parallel_unary_numpy_apply_with_function() -> None:
-    with tempfile.TemporaryDirectory() as tempdir:
-        path1 = os.path.join(tempdir, "test1.tif")
-        data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
-        dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
-        dataset1.Close()
-        layer1 = RasterLayer.layer_from_file(path1)
-
-        result = RasterLayer.empty_raster_layer_like(layer1)
-
-        def simple_add(chunk):
-            return chunk + 1.0
-
-        comp = layer1.numpy_apply(simple_add)
-        comp.ystep = 1
-        comp.parallel_save(result)
-
-        expected = data1 + 1.0
-        actual = result.read_array(0, 0, 4, 2)
-
-        assert (expected == actual).all()
-
-@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
-def test_parallel_unary_numpy_apply_with_lambda() -> None:
-    with tempfile.TemporaryDirectory() as tempdir:
-        path1 = os.path.join(tempdir, "test1.tif")
-        data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
-        dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
-        dataset1.Close()
-        layer1 = RasterLayer.layer_from_file(path1)
-
-        result = RasterLayer.empty_raster_layer_like(layer1)
-
-        comp = layer1.numpy_apply(lambda a: a + 1.0)
-        comp.ystep = 1
-        comp.parallel_save(result)
-
-        expected = data1 + 1.0
-        actual = result.read_array(0, 0, 4, 2)
-
-        assert (expected == actual).all()
-
-@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
-def test_parallel_where_simple() -> None:
-    with tempfile.TemporaryDirectory() as tempdir:
-        path1 = os.path.join(tempdir, "test1.tif")
-        data1 = np.array([[0, 1, 0, 2], [0, 0, 1, 1]])
-        dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
-        dataset1.Close()
-        layer1 = RasterLayer.layer_from_file(path1)
-
-        result = RasterLayer.empty_raster_layer_like(layer1)
-
-        comp = LayerOperation.where(layer1 > 0, 1, 2)
-        comp.ystep = 1
-        comp.parallel_save(result)
-
-        expected = np.where(data1 > 0, 1, 2)
-        actual = result.read_array(0, 0, 4, 2)
-        assert (expected == actual).all()
-
-@pytest.mark.skipif(yirgacheffe.backends.BACKEND != "NUMPY", reason="Only applies for numpy")
-def test_parallel_conv2d() -> None:
-    data1 = np.array([
-        [0, 0, 0, 0, 0],
-        [0, 1, 1, 1, 0],
-        [0, 1, 1, 1, 0],
-        [0, 1, 1, 1, 0],
-        [0, 0, 0, 0, 0],
-    ]).astype(np.float32)
-    weights = np.array([
-        [0.0, 0.1, 0.0],
-        [0.1, 0.6, 0.1],
-        [0.0, 0.1, 0.0],
-    ])
-
-    conv = torch.nn.Conv2d(1, 1, 3, padding=1, bias=False)
-    conv.weight = torch.nn.Parameter(torch.from_numpy(np.array([[weights.astype(np.float32)]])))
-    tensorres = conv(torch.from_numpy(np.array([[data1]])))
-    expected = tensorres.detach().numpy()[0][0]
-
-    with RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1)) as layer1:
-
-        calc = layer1.conv2d(weights)
-        calc.ystep = 1
-        with RasterLayer.empty_raster_layer_like(layer1) as res:
-            calc.save(res)
-            actual = res.read_array(0, 0, 5, 5)
-
-            # Torch and MLX give slightly different rounding
-            assert np.isclose(expected, actual).all()

yirgacheffe-1.3.1/yirgacheffe/constants.py

@@ -1 +0,0 @@
-YSTEP = 512