yirgacheffe 1.2.0__tar.gz → 1.3.0__tar.gz

{yirgacheffe-1.2.0/yirgacheffe.egg-info → yirgacheffe-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.2.0
+Version: 1.3.0
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC
@@ -35,6 +35,14 @@ Example common use-cases:
 * Do the raster layers get big and take up large amounts of memory? Yirgacheffe will let you do simple numerical operations with layers directly and then worry about the memory management behind the scenes for you.
 
 
+## Installation
+
+Yirgacheffe is available via pypi, so can be installed with pip for example:
+
+```SystemShell
+$ pip install yirgacheffe
+```
+
 ## Basic usage
 
 They main unit of data in Yirgacheffe is a "layer", which wraps either a raster dataset or polygon data, and then you can do work on layers without having to worry (unless you choose to) about how they align - Yirgacheffe will work out all the details around overlapping
@@ -295,6 +303,7 @@ with RasterLayer.layer_from_file('test1.tif') as layer1:
 
 The following math operators common to numpy and other libraries are currently supported:
 
+* abs
 * clip
 * exp
 * exp2
@@ -338,6 +347,18 @@ with RasterLayer.layer_from_file('original.tif') as layer1:
         calc.save(result)
 ```
 
+### Type conversion
+
+Similar to numpy and other Python numerical libraries, Yirgacheffe will automatically deal with simple type conversion where possible, however sometimes explicit conversion is either necessary or desired. Similar to numpy, there is an `astype` operator that lets you set the conversion:
+
+```python
+from yirgacheffe.operations import DataType
+
+
+with RasterLayer.layer_from_file('float_data.tif') as float_layer:
+    int_layer = float_layer.astype(DataType.Int32)
+```
+
 ### Apply
 
 You can specify a function that takes either data from one layer or from two layers, and returns the processed data. There's two version of this: one that lets you specify a numpy function that'll be applied to the layer data as an array, or one that is more shader like that lets you do pixel wise processing.

{yirgacheffe-1.2.0 → yirgacheffe-1.3.0}/README.md

@@ -11,6 +11,14 @@ Example common use-cases:
 * Do the raster layers get big and take up large amounts of memory? Yirgacheffe will let you do simple numerical operations with layers directly and then worry about the memory management behind the scenes for you.
 
 
+## Installation
+
+Yirgacheffe is available via pypi, so can be installed with pip for example:
+
+```SystemShell
+$ pip install yirgacheffe
+```
+
 ## Basic usage
 
 They main unit of data in Yirgacheffe is a "layer", which wraps either a raster dataset or polygon data, and then you can do work on layers without having to worry (unless you choose to) about how they align - Yirgacheffe will work out all the details around overlapping
@@ -271,6 +279,7 @@ with RasterLayer.layer_from_file('test1.tif') as layer1:
 
 The following math operators common to numpy and other libraries are currently supported:
 
+* abs
 * clip
 * exp
 * exp2
@@ -314,6 +323,18 @@ with RasterLayer.layer_from_file('original.tif') as layer1:
         calc.save(result)
 ```
 
+### Type conversion
+
+Similar to numpy and other Python numerical libraries, Yirgacheffe will automatically deal with simple type conversion where possible, however sometimes explicit conversion is either necessary or desired. Similar to numpy, there is an `astype` operator that lets you set the conversion:
+
+```python
+from yirgacheffe.operations import DataType
+
+
+with RasterLayer.layer_from_file('float_data.tif') as float_layer:
+    int_layer = float_layer.astype(DataType.Int32)
+```
+
 ### Apply
 
 You can specify a function that takes either data from one layer or from two layers, and returns the processed data. There's two version of this: one that lets you specify a numpy function that'll be applied to the layer data as an array, or one that is more shader like that lets you do pixel wise processing.

{yirgacheffe-1.2.0 → yirgacheffe-1.3.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "yirgacheffe"
-version = "1.2.0"
+version = "1.3.0"
 description = "Abstraction of gdal datasets for doing basic math operations"
 readme = "README.md"
 authors = [{ name = "Michael Dales", email = "mwd24@cam.ac.uk" }]

yirgacheffe-1.3.0/tests/test_datatypes.py

@@ -0,0 +1,67 @@
+
+import numpy as np
+import pytest
+from osgeo import gdal
+
+from yirgacheffe.operators import DataType
+from yirgacheffe.backends import backend, BACKEND
+from yirgacheffe.layers import RasterLayer
+
+from helpers import gdal_dataset_with_data
+
+@pytest.mark.parametrize("gtype", [
+	gdal.GDT_Int8,
+	gdal.GDT_Int16,
+	gdal.GDT_Int32,
+	gdal.GDT_Int64,
+	gdal.GDT_Byte,
+	gdal.GDT_UInt16,
+	gdal.GDT_UInt32,
+	gdal.GDT_UInt64,
+	gdal.GDT_Float32,
+])
+def test_round_trip(gtype) -> None:
+	ytype = DataType.of_gdal(gtype)
+	backend_type = backend.dtype_to_backed(ytype)
+	assert backend.backend_to_dtype(backend_type) == ytype
+
+@pytest.mark.parametrize("ytype", [
+	DataType.Int8,
+	DataType.Int16,
+	DataType.Int32,
+	DataType.Int64,
+	DataType.UInt8,
+	DataType.UInt16,
+	DataType.UInt32,
+	DataType.UInt64,
+	DataType.Float32,
+	DataType.Float64,
+])
+def test_round_trip_from_gdal(ytype) -> None:
+	gtype = ytype.to_gdal()
+	assert DataType.of_gdal(gtype) == ytype
+
+def test_round_trip_float64() -> None:
+	backend_type = backend.dtype_to_backed(DataType.Float64)
+	ytype = backend.backend_to_dtype(backend_type)
+	print(BACKEND, "sad")
+	match BACKEND:
+		case "NUMPY":
+			assert ytype == DataType.Float64
+		case "MLX":
+			assert ytype == DataType.Float32
+
+def test_float_to_int() -> None:
+	data1 = np.array([[1.0, 2.0, 3.0, 4.0], [5.5, 6.5, 7.5, 8.5]])
+
+	layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
+	# Note the float 32 here is to rule out that writing the result to the
+	# new dataset was what caused the truncation
+	result = RasterLayer.empty_raster_layer_like(layer1, datatype=DataType.Float32)
+
+	comp = layer1.astype(DataType.UInt8)
+	comp.save(result)
+
+	expected = backend.promote(np.array([[1, 2, 3, 4], [5, 6, 7, 8]]))
+	actual = backend.demote_array(result.read_array(0, 0, 4, 2))
+	assert (expected == actual).all()

{yirgacheffe-1.2.0 → yirgacheffe-1.3.0}/tests/test_operators.py

@@ -10,7 +10,7 @@ from osgeo import gdal
 from helpers import gdal_dataset_with_data
 import yirgacheffe
 from yirgacheffe.layers import RasterLayer, ConstantLayer
-from yirgacheffe.operators import LayerOperation
+from yirgacheffe.operators import LayerOperation, DataType
 from yirgacheffe.backends import backend
 
 def test_add_byte_layers() -> None:
@@ -21,8 +21,8 @@ def test_add_byte_layers() -> None:
     layer2 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data2))
     result = RasterLayer.empty_raster_layer_like(layer1)
 
-    assert layer1.datatype == gdal.GDT_Byte
-    assert layer2.datatype == gdal.GDT_Byte
+    assert layer1.datatype == DataType.Byte
+    assert layer2.datatype == DataType.Byte
 
     comp = layer1 + layer2
     comp.save(result)
@@ -64,8 +64,8 @@ def test_add_byte_layers_with_callback(skip, expected_steps) -> None:
     layer2 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data2))
     result = RasterLayer.empty_raster_layer_like(layer1)
 
-    assert layer1.datatype == gdal.GDT_Byte
-    assert layer2.datatype == gdal.GDT_Byte
+    assert layer1.datatype == DataType.Byte
+    assert layer2.datatype == DataType.Byte
 
     callback_possitions = []
 
@@ -632,7 +632,7 @@ def test_save_and_sum_float32(monkeypatch) -> None:
 
     data1 = np.array(data, dtype=np.float32)
     layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
-    assert layer1.datatype == gdal.GDT_Float32
+    assert layer1.datatype == DataType.Float32
 
     # Sum forces things to float64
     expected = np.sum(data1.astype(np.float64))
@@ -660,7 +660,7 @@ def test_parallel_save_and_sum_float32(monkeypatch) -> None:
         dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
         dataset1.Close()
         layer1 = RasterLayer.layer_from_file(path1)
-        assert layer1.datatype == gdal.GDT_Float32
+        assert layer1.datatype == DataType.Float32
 
         # Sum forces things to float64
         expected = np.sum(data1.astype(np.float64))
@@ -684,7 +684,7 @@ def test_sum_float32(monkeypatch) -> None:
 
     data1 = np.array(data, dtype=np.float32)
     layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
-    assert layer1.datatype == gdal.GDT_Float32
+    assert layer1.datatype == DataType.Float32
 
     # Sum forces things to float64
     expected = np.sum(data1.astype(np.float64))
@@ -703,8 +703,8 @@ def test_and_byte_layers() -> None:
     layer2 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data2))
     result = RasterLayer.empty_raster_layer_like(layer1)
 
-    assert layer1.datatype == gdal.GDT_Byte
-    assert layer2.datatype == gdal.GDT_Byte
+    assert layer1.datatype == DataType.Byte
+    assert layer2.datatype == DataType.Byte
 
     comp = layer1 & layer2
     comp.save(result)
@@ -722,8 +722,8 @@ def test_or_byte_layers() -> None:
     layer2 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data2))
     result = RasterLayer.empty_raster_layer_like(layer1)
 
-    assert layer1.datatype == gdal.GDT_Byte
-    assert layer2.datatype == gdal.GDT_Byte
+    assert layer1.datatype == DataType.Byte
+    assert layer2.datatype == DataType.Byte
 
     comp = layer1 | layer2
     comp.save(result)
@@ -741,8 +741,8 @@ def test_and_int_layers() -> None:
     layer2 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data2))
     result = RasterLayer.empty_raster_layer_like(layer1)
 
-    assert layer1.datatype == gdal.GDT_Int16
-    assert layer2.datatype == gdal.GDT_Int16
+    assert layer1.datatype == DataType.Int16
+    assert layer2.datatype == DataType.Int16
 
     comp = layer1 & layer2
     comp.save(result)
@@ -760,8 +760,8 @@ def test_or_int_layers() -> None:
     layer2 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data2))
     result = RasterLayer.empty_raster_layer_like(layer1)
 
-    assert layer1.datatype == gdal.GDT_Int16
-    assert layer2.datatype == gdal.GDT_Int16
+    assert layer1.datatype == DataType.Int16
+    assert layer2.datatype == DataType.Int16
 
     comp = layer1 | layer2
     comp.save(result)
@@ -1197,6 +1197,34 @@ def test_clip_lower_module() -> None:
     actual = result.read_array(0, 0, 4, 2)
     assert (expected == actual).all()
 
+def test_abs_method() -> None:
+    data1 = np.array([[1.0, 2.0, 3.0, 4.0], [-1.0, -2.0, -3.0, -4.0]])
+    layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
+    result = RasterLayer.empty_raster_layer_like(layer1)
+
+    comp = layer1.abs()
+    comp.save(result)
+
+    expected = backend.abs_op(backend.promote(data1))
+    backend.eval_op(expected)
+
+    actual = result.read_array(0, 0, 4, 2)
+    assert (expected == actual).all()
+
+def test_abs_module() -> None:
+    data1 = np.array([[1.0, 2.0, 3.0, 4.0], [-1.0, -2.0, -3.0, -4.0]])
+    layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 0.02, data1))
+    result = RasterLayer.empty_raster_layer_like(layer1)
+
+    comp = LayerOperation.abs(layer1)
+    comp.save(result)
+
+    expected = backend.abs_op(backend.promote(data1))
+    backend.eval_op(expected)
+
+    actual = result.read_array(0, 0, 4, 2)
+    assert (expected == actual).all()
+
 @pytest.mark.parametrize("skip", [
     1,
     2,

{yirgacheffe-1.2.0 → yirgacheffe-1.3.0}/tests/test_raster.py

@@ -9,6 +9,7 @@ from helpers import gdal_dataset_of_region, gdal_multiband_dataset_with_data
 from yirgacheffe.window import Area, PixelScale, Window
 from yirgacheffe.layers import RasterLayer, InvalidRasterBand
 from yirgacheffe.rounding import round_up_pixels
+from yirgacheffe.operators import DataType
 
 
 # There is a lot of "del" in this file, due to a combination of gdal having no way
@@ -128,12 +129,12 @@ def test_empty_layer_from_raster_with_new_smaller_area():
 
 def test_empty_layer_from_raster_new_datatype():
     source = RasterLayer(gdal_dataset_of_region(Area(-10, 10, 10, -10), 0.02))
-    assert source.datatype == gdal.GDT_Byte
+    assert source.datatype == DataType.Byte
     empty = RasterLayer.empty_raster_layer_like(source, datatype=gdal.GDT_Float64)
     assert empty.pixel_scale == source.pixel_scale
     assert empty.projection == source.projection
     assert empty.window == source.window
-    assert empty.datatype == gdal.GDT_Float64
+    assert empty.datatype == DataType.Float64
 
 def test_empty_layer_from_raster_with_window():
     source = RasterLayer(gdal_dataset_of_region(Area(-10, 10, 10, -10), 0.02))

{yirgacheffe-1.2.0 → yirgacheffe-1.3.0}/tests/test_vectors.py

@@ -8,6 +8,7 @@ from helpers import make_vectors_with_mutlile_ids, make_vectors_with_id, make_ve
 from yirgacheffe import WGS_84_PROJECTION
 from yirgacheffe.layers import RasterLayer, RasteredVectorLayer, VectorLayer, VectorRangeLayer, DynamicVectorRangeLayer
 from yirgacheffe.window import Area, PixelScale, Window
+from yirgacheffe.operators import DataType
 
 def test_basic_vector_layer_no_filter_match() -> None:
     with tempfile.TemporaryDirectory() as tempdir:
@@ -139,7 +140,7 @@ def test_vector_layers_with_default_burn_value(klass) -> None:
         assert layer.area == Area(-10.0, 10.0, 10.0, -10.0)
         assert layer.geo_transform == (-10.0, 1.0, 0.0, 10.0, 0.0, -1.0)
         assert layer.window == Window(0, 0, 20, 20)
-        assert layer.datatype == gdal.GDT_Byte
+        assert layer.datatype == DataType.Byte
 
         # The default burn value is 1, so check that if we sum the area
         # we get half and half
@@ -256,13 +257,13 @@ def test_vector_layers_with_field_value(klass) -> None:
 @pytest.mark.parametrize(
     "value,expected",
     [
-        (1, gdal.GDT_Byte),
-        (42, gdal.GDT_Byte),
-        (-1, gdal.GDT_Int16),
-        (1024, gdal.GDT_UInt16),
-        (1024*1024, gdal.GDT_UInt32),
-        (-1024*1024, gdal.GDT_Int32),
-        (1.0, gdal.GDT_Float64),
+        (1, DataType.Byte),
+        (42, DataType.Byte),
+        (-1, DataType.Int16),
+        (1024, DataType.UInt16),
+        (1024*1024, DataType.UInt32),
+        (-1024*1024, DataType.Int32),
+        (1.0, DataType.Float64),
     ]
 )
 def test_vector_layers_with_guessed_type_burn_value(value, expected) -> None:
@@ -294,14 +295,14 @@ def test_vector_layers_with_guessed_type_burn_value(value, expected) -> None:
 @pytest.mark.parametrize(
     "value,datatype",
     [
-        (1, gdal.GDT_Byte),
-        (42, gdal.GDT_Byte),
-        (1, gdal.GDT_Int16),
-        (42, gdal.GDT_Int16),
-        (1024, gdal.GDT_Int16),
-        (1.0, gdal.GDT_Float32),
-        (0.5, gdal.GDT_Float32),
-        (1.0, gdal.GDT_Float64),
+        (1, DataType.Byte),
+        (42, DataType.Byte),
+        (1, DataType.Int16),
+        (42, DataType.Int16),
+        (1024, DataType.Int16),
+        (1.0, DataType.Float32),
+        (0.5, DataType.Float32),
+        (1.0, DataType.Float64),
     ]
 )
 def test_vector_layers_with_different_type_burn_value(value, datatype) -> None:
@@ -334,8 +335,8 @@ def test_vector_layers_with_different_type_burn_value(value, datatype) -> None:
 @pytest.mark.parametrize(
     "value,expected",
     [
-        (1, gdal.GDT_Int64),
-        (1.0, gdal.GDT_Float64),
+        (1, DataType.Int64),
+        (1.0, DataType.Float64),
     ]
 )
 def test_vector_layers_with_guess_field_type_burn_value(value, expected) -> None:
@@ -448,7 +449,7 @@ def test_vector_layers_with_empty_features(klass) -> None:
         assert layer.area == Area(-10.0, 10.0, 10.0, -10.0)
         assert layer.geo_transform == (-10.0, 1.0, 0.0, 10.0, 0.0, -1.0)
         assert layer.window == Window(0, 0, 20, 20)
-        assert layer.datatype == gdal.GDT_Byte
+        assert layer.datatype == DataType.Byte
 
         # The default burn value is 1, so check that if we sum the area
         # we get half and half

yirgacheffe-1.3.0/yirgacheffe/backends/enumeration.py

@@ -0,0 +1,58 @@
+from enum import Enum
+
+from osgeo import gdal
+
+class operators(Enum):
+    ADD = 1
+    SUB = 2
+    MUL = 3
+    TRUEDIV = 4
+    POW = 5
+    EQ = 6
+    NE = 7
+    LT = 8
+    LE = 9
+    GT = 10
+    GE = 11
+    AND = 12
+    OR = 13
+    LOG = 14
+    LOG2 = 15
+    LOG10 = 16
+    EXP = 17
+    EXP2 = 18
+    CLIP = 19
+    WHERE = 20
+    MIN = 21
+    MAX = 22
+    SUM = 23
+    MINIMUM = 24
+    MAXIMUM = 25
+    NAN_TO_NUM = 26
+    ISIN = 27
+    REMAINDER = 28
+    FLOORDIV = 29
+    CONV2D = 30
+    ABS = 31
+    ASTYPE = 32
+
+
+class dtype(Enum):
+    Float32 = gdal.GDT_Float32
+    Float64 = gdal.GDT_Float64
+    Byte = gdal.GDT_Byte
+    Int8 = gdal.GDT_Int8
+    Int16 = gdal.GDT_Int16
+    Int32 = gdal.GDT_Int32
+    Int64 = gdal.GDT_Int64
+    UInt8 = gdal.GDT_Byte
+    UInt16 = gdal.GDT_UInt16
+    UInt32 = gdal.GDT_UInt32
+    UInt64 = gdal.GDT_UInt64
+
+    def to_gdal(self):
+        return self.value
+
+    @classmethod
+    def of_gdal(cls, val):
+        return cls(val)

{yirgacheffe-1.2.0 → yirgacheffe-1.3.0}/yirgacheffe/backends/mlx.py

@@ -4,6 +4,7 @@ import mlx.core as mx # pylint: disable=E0001,E0611,E0401
 import mlx.nn
 
 from .enumeration import operators as op
+from .enumeration import dtype
 
 array_t = mx.array
 float_t = mx.float32
@@ -43,6 +44,7 @@ full = mx.full
 allclose = mx.allclose
 remainder_op = mx.remainder
 floordiv_op = mx.array.__floordiv__
+abs_op = mx.abs
 
 def sum_op(a):
     # There are weird issues around how MLX overflows int8, so just promote the data ahead of summing
@@ -120,9 +122,63 @@ def conv2d_op(data, weights):
     preped_data = mx.array(np.reshape(unshifted_data_shape, conv_data_shape))
 
     shifted_res = conv(preped_data)[0]
-    res = np.reshape(shifted_res, [1] + list(shifted_res.shape)[:-1])
+    res = mx.reshape(shifted_res, [1] + list(shifted_res.shape)[:-1])
     return res[0]
 
+
+def dtype_to_backed(dt):
+    match dt:
+        case dtype.Float32:
+            return mx.float32
+        case dtype.Float64:
+            return mx.float32
+        case dtype.Byte:
+            return mx.uint8
+        case dtype.Int8:
+            return mx.int8
+        case dtype.Int16:
+            return mx.int16
+        case dtype.Int32:
+            return mx.int32
+        case dtype.Int64:
+            return mx.int64
+        case dtype.UInt8:
+            return mx.uint8
+        case dtype.UInt16:
+            return mx.uint16
+        case dtype.UInt32:
+            return mx.uint32
+        case dtype.UInt64:
+            return mx.uint64
+        case _:
+            raise ValueError
+
+def backend_to_dtype(val):
+    match val:
+        case mx.float32:
+            return dtype.Float32
+        case mx.int8:
+            return dtype.Int8
+        case mx.int16:
+            return dtype.Int16
+        case mx.int32:
+            return dtype.Int32
+        case mx.int64:
+            return dtype.Int64
+        case mx.uint8:
+            return dtype.Byte
+        case mx.uint16:
+            return dtype.UInt16
+        case mx.uint32:
+            return dtype.UInt32
+        case mx.uint64:
+            return dtype.UInt64
+        case _:
+            raise ValueError
+
+def astype_op(data, datatype):
+    return data.astype(dtype_to_backed(datatype))
+
 operator_map = {
     op.ADD: mx.array.__add__,
     op.SUB: mx.array.__sub__,
@@ -153,4 +209,6 @@ operator_map = {
     op.REMAINDER: mx.remainder,
     op.FLOORDIV: mx.array.__floordiv__,
     op.CONV2D: conv2d_op,
+    op.ABS: mx.abs,
+    op.ASTYPE: astype_op,
 }