yirgacheffe 1.7.5__tar.gz → 1.7.7__tar.gz

{yirgacheffe-1.7.5/yirgacheffe.egg-info → yirgacheffe-1.7.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.7.5
+Version: 1.7.7
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC
@@ -262,17 +262,33 @@ Notes:
 * You can have missing tiles, and these will be filled in with zeros.
 * You can have tiles that overlap, so long as they still conform to the rule that all tiles are the same size and on a grid.
 
-### ConstantLayer
+### Constants
 
-This is there to simplify code when you have some optional layers. Rather than littering your code with checks, you can just use a constant layer, which can be included in calculations and will just return an fixed value as if it wasn't there. Useful with 0.0 or 1.0 for sum or multiplication null layers.
+At times it is useful to have a fixed constant in an expression. Typically, similar to numpy, if an expression involving layers has a constant in, Yirgacheffe will apply that to all pixels in the equation without need for further elaboration:
+
+```python
+with yg.read_raster("some_data.tif") as layer:
+    doubled_layer = layer * 2.0
+    ...
+```
+
+This can be useful in tasks where you have an optional layer in your code. For example, here the code optionally loads an area-per-pixel layer, which if not present can just be substituted with a 1.0:
 
 ```python
 try:
-    area_layer = RasterLayer.layer_from_file('myarea.tiff')
+    area_layer = yg.read_raster('myarea.tiff')
 except FileDoesNotExist:
-    area_layer = ConstantLayer(0.0)
+    area_layer = 1.0
 ```
 
+However, as with numpy, Python can not make the correct inference if the constant value is the first term in the equation. In that case you need to explicitly wrap the value with `constant` to help Python understand what is happening:
+
+```python
+with yg.read_raster("some_data.tif") as layer:
+    result = yg.constant(1.0) / layer
+```
+
+
 ### H3CellLayer
 
 If you have H3 installed, you can generate a mask layer based on an H3 cell identifier, where pixels inside the cell will have a value of 1, and those outside will have a value of 0.

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/README.md

@@ -232,17 +232,33 @@ Notes:
 * You can have missing tiles, and these will be filled in with zeros.
 * You can have tiles that overlap, so long as they still conform to the rule that all tiles are the same size and on a grid.
 
-### ConstantLayer
+### Constants
 
-This is there to simplify code when you have some optional layers. Rather than littering your code with checks, you can just use a constant layer, which can be included in calculations and will just return an fixed value as if it wasn't there. Useful with 0.0 or 1.0 for sum or multiplication null layers.
+At times it is useful to have a fixed constant in an expression. Typically, similar to numpy, if an expression involving layers has a constant in, Yirgacheffe will apply that to all pixels in the equation without need for further elaboration:
+
+```python
+with yg.read_raster("some_data.tif") as layer:
+    doubled_layer = layer * 2.0
+    ...
+```
+
+This can be useful in tasks where you have an optional layer in your code. For example, here the code optionally loads an area-per-pixel layer, which if not present can just be substituted with a 1.0:
 
 ```python
 try:
-    area_layer = RasterLayer.layer_from_file('myarea.tiff')
+    area_layer = yg.read_raster('myarea.tiff')
 except FileDoesNotExist:
-    area_layer = ConstantLayer(0.0)
+    area_layer = 1.0
 ```
 
+However, as with numpy, Python can not make the correct inference if the constant value is the first term in the equation. In that case you need to explicitly wrap the value with `constant` to help Python understand what is happening:
+
+```python
+with yg.read_raster("some_data.tif") as layer:
+    result = yg.constant(1.0) / layer
+```
+
+
 ### H3CellLayer
 
 If you have H3 installed, you can generate a mask layer based on an H3 cell identifier, where pixels inside the cell will have a value of 1, and those outside will have a value of 0.

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "yirgacheffe"
-version = "1.7.5"
+version = "1.7.7"
 description = "Abstraction of gdal datasets for doing basic math operations"
 readme = "README.md"
 authors = [{ name = "Michael Dales", email = "mwd24@cam.ac.uk" }]

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/tests/test_auto_windowing.py

@@ -201,27 +201,91 @@ def test_constant_layer_result_lhs_multiply() -> None:
 
 def test_vector_layers_add() -> None:
     data1 = np.array([[1, 2], [3, 4]])
-    layer1 = RasterLayer(gdal_dataset_with_data((0.0, 0.0), 1.0, data1))
+    with RasterLayer(gdal_dataset_with_data((0.0, 0.0), 1.1, data1)) as raster_layer:
+        with tempfile.TemporaryDirectory() as tempdir:
+            path = os.path.join(tempdir, "test.gpkg")
+            areas = {
+                (Area(-10.0, 10.0, 0.0, 0.0), 42),
+                (Area(0.0, 0.0, 10, -10), 43)
+            }
+            make_vectors_with_mutlile_ids(areas, path)
+
+            burn_value = 2
+            with VectorLayer.layer_from_file(
+                path,
+                None,
+                raster_layer.map_projection.scale,
+                raster_layer.map_projection.name,
+                burn_value=burn_value
+            ) as vector_layer:
+                layer2_total = vector_layer.sum()
+                assert layer2_total == ((vector_layer.window.xsize * vector_layer.window.ysize) / 2) * burn_value
+
+                calc = raster_layer + vector_layer
+
+                assert calc.area == vector_layer.area
+
+                total = calc.sum()
+                assert total == layer2_total + np.sum(data1)
+
+                with RasterLayer.empty_raster_layer_like(calc) as result:
+                    calc.save(result)
+                    total = result.sum()
+                    assert total == layer2_total + np.sum(data1)
+
+def test_vector_layers_add_unbound_rhs() -> None:
+    data1 = np.array([[1, 2], [3, 4]])
+    with RasterLayer(gdal_dataset_with_data((0.0, 0.0), 1.1, data1)) as raster_layer:
+        with tempfile.TemporaryDirectory() as tempdir:
+            path = os.path.join(tempdir, "test.gpkg")
+            areas = {
+                (Area(-10.0, 10.0, 0.0, 0.0), 42),
+                (Area(0.0, 0.0, 10, -10), 43)
+            }
+            make_vectors_with_mutlile_ids(areas, path)
 
-    with tempfile.TemporaryDirectory() as tempdir:
-        path = os.path.join(tempdir, "test.gpkg")
-        areas = {
-            (Area(-10.0, 10.0, 0.0, 0.0), 42),
-            (Area(0.0, 0.0, 10, -10), 43)
-        }
-        make_vectors_with_mutlile_ids(areas, path)
+            burn_value = 2
+            with VectorLayer.layer_from_file(path, None, None, None, burn_value=burn_value) as vector_layer:
+                calc = raster_layer + vector_layer
 
-        burn_value = 2
-        layer2 = VectorLayer.layer_from_file(path, None, layer1.pixel_scale, layer1.projection, burn_value=burn_value)
-        layer2_total = layer2.sum()
-        assert layer2_total == ((layer2.window.xsize * layer2.window.ysize) / 2) * burn_value
+                layer2_total = ((calc.window.xsize * calc.window.ysize) / 2) * burn_value
 
-        calc = layer1 + layer2
+                assert calc.area != vector_layer.area
 
-        assert calc.area == layer2.area
+                total = calc.sum()
+                assert total == layer2_total + np.sum(data1)
 
-        total = calc.sum()
-        assert total == layer2_total + np.sum(data1)
+                with RasterLayer.empty_raster_layer_like(calc) as result:
+                    calc.save(result)
+                    total = result.sum()
+                    assert total == layer2_total + np.sum(data1)
+
+def test_vector_layers_add_unbound_lhs() -> None:
+    data1 = np.array([[1, 2], [3, 4]])
+    with RasterLayer(gdal_dataset_with_data((0.0, 0.0), 1.1, data1)) as raster_layer:
+        with tempfile.TemporaryDirectory() as tempdir:
+            path = os.path.join(tempdir, "test.gpkg")
+            areas = {
+                (Area(-10.0, 10.0, 0.0, 0.0), 42),
+                (Area(0.0, 0.0, 10, -10), 43)
+            }
+            make_vectors_with_mutlile_ids(areas, path)
+
+            burn_value = 2
+            with VectorLayer.layer_from_file(path, None, None, None, burn_value=burn_value) as vector_layer:
+                calc = vector_layer + raster_layer
+
+                layer2_total = ((calc.window.xsize * calc.window.ysize) / 2) * burn_value
+
+                assert calc.area != vector_layer.area
+
+                total = calc.sum()
+                assert total == layer2_total + np.sum(data1)
+
+                with RasterLayer.empty_raster_layer_like(calc) as result:
+                    calc.save(result)
+                    total = result.sum()
+                    assert total == layer2_total + np.sum(data1)
 
 def test_vector_layers_multiply() -> None:
     data1 = np.array([[1, 2], [3, 4]])

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/tests/test_openers.py

@@ -1,5 +1,6 @@
 import os
 import tempfile
+from math import ceil, floor
 from pathlib import Path
 
 import numpy as np
@@ -7,8 +8,9 @@ import pytest
 
 import yirgacheffe as yg
 from yirgacheffe import WGS_84_PROJECTION
-from yirgacheffe.layers import InvalidRasterBand
+from yirgacheffe.layers import InvalidRasterBand, RasterLayer
 from yirgacheffe.window import Area, MapProjection, Window
+from yirgacheffe.operators import DataType
 from tests.helpers import gdal_dataset_of_region, gdal_multiband_dataset_with_data, \
     make_vectors_with_id, make_vectors_with_mutlile_ids
 
@@ -213,3 +215,54 @@ def test_open_two_raster_by_glob(tiled):
             with yg.read_raster(path1) as raster1:
                 with yg.read_raster(path2) as raster2:
                     assert group.sum() == raster1.sum() + raster2.sum()
+
+def test_open_uniform_area_layer() -> None:
+    with tempfile.TemporaryDirectory() as tempdir:
+        path = os.path.join(tempdir, "test.tif")
+        pixel_scale = 0.5
+        area = Area(
+            floor(-180 / pixel_scale) * pixel_scale,
+            ceil(90 / pixel_scale) * pixel_scale,
+            (floor(-180 / pixel_scale) * pixel_scale) + pixel_scale,
+            floor(-90 / pixel_scale) * pixel_scale
+        )
+        dataset = gdal_dataset_of_region(area, pixel_scale, filename=path)
+        assert dataset.RasterXSize == 1
+        assert dataset.RasterYSize == ceil(180 / pixel_scale)
+        dataset.Close()
+
+        with yg.read_narrow_raster(path) as layer:
+            assert layer.map_projection is not None
+            assert layer.map_projection.scale == (pixel_scale, -pixel_scale)
+            assert layer.area == Area(
+                floor(-180 / pixel_scale) * pixel_scale,
+                ceil(90 / pixel_scale) * pixel_scale,
+                ceil(180 / pixel_scale) * pixel_scale,
+                floor(-90 / pixel_scale) * pixel_scale
+            )
+            assert layer.window == Window(
+                0,
+                0,
+                ceil((layer.area.right - layer.area.left) / pixel_scale),
+                ceil((layer.area.top - layer.area.bottom) / pixel_scale)
+            )
+
+def test_incorrect_tiff_for_uniform_area() -> None:
+    with tempfile.TemporaryDirectory() as tempdir:
+        path = Path(tempdir) / "test.tif"
+        area = Area(-10, 10, 10, -10)
+        gdal_dataset_of_region(area, 1.0, filename=path)
+        assert path.exists()
+        with pytest.raises(ValueError):
+            _ = yg.read_narrow_raster(path)
+
+def test_constant() -> None:
+    with yg.constant(42.0) as layer:
+        area = Area(left=-1.0, right=1.0, top=1.0, bottom=-1.0)
+        projection = MapProjection(WGS_84_PROJECTION, 0.1, -0.1)
+        with RasterLayer.empty_raster_layer(area, projection.scale, DataType.Float32) as result:
+            layer.save(result)
+
+            expected = np.full((20, 20), 42.0)
+            actual = result.read_array(0, 0, 20, 20)
+            assert (expected == actual).all()

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/tests/test_operators.py

@@ -10,7 +10,8 @@ import torch
 import yirgacheffe
 from yirgacheffe.window import Area, PixelScale
 from yirgacheffe.layers import ConstantLayer, RasterLayer, VectorLayer
-from yirgacheffe.operators import LayerOperation, DataType
+from yirgacheffe.operators import DataType
+from yirgacheffe._operators import LayerOperation
 from yirgacheffe._backends import backend
 from tests.helpers import gdal_dataset_with_data, gdal_dataset_of_region, make_vectors_with_id
 

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/tests/test_parallel_operators.py

@@ -1,4 +1,5 @@
 import os
+import resource
 import tempfile
 
 import numpy as np
@@ -8,7 +9,7 @@ import torch
 import yirgacheffe
 from tests.helpers import gdal_dataset_with_data
 from yirgacheffe.layers import RasterLayer
-from yirgacheffe.operators import LayerOperation
+from yirgacheffe._operators import LayerOperation
 
 # These tests are marked skip for MLX, because there seems to be a problem with
 # calling mx.eval in the tests for parallel save on Linux (which is what we use
@@ -42,7 +43,6 @@ def test_add_byte_layers_with_one_thread_uses_regular_save(monkeypatch) -> None:
             with pytest.raises(TypeError):
                 comp.parallel_save(result)
 
-
 @pytest.mark.skipif(yirgacheffe._backends.BACKEND != "NUMPY", reason="Only applies for numpy")
 def test_add_byte_layers(monkeypatch) -> None:
     with monkeypatch.context() as m:
@@ -71,6 +71,39 @@ def test_add_byte_layers(monkeypatch) -> None:
 
             assert (expected == actual).all()
 
+@pytest.mark.skipif(yirgacheffe._backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_rlimit_nofiles(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+
+            rlimit_log = []
+            def callback_rlimit_recorder(_progress: float) -> None:
+                rlimit_log.append(resource.getrlimit(resource.RLIMIT_NOFILE))
+
+            with RasterLayer.layer_from_file(path1) as layer:
+                with RasterLayer.empty_raster_layer_like(layer) as result:
+
+                    before_current_fd_limit, before_max_fd_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+
+                    comp = layer * 2
+                    comp.parallel_save(result, callback=callback_rlimit_recorder)
+
+                    after_current_fd_limit, after_max_fd_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+
+                    assert after_current_fd_limit == before_current_fd_limit
+                    assert after_max_fd_limit == before_max_fd_limit
+
+                    assert len(rlimit_log) > 0
+                    for recorded_current_fd_limit, recorded_max_fd_limit in rlimit_log:
+                        assert recorded_current_fd_limit == before_max_fd_limit
+                        assert recorded_max_fd_limit == before_max_fd_limit
+
 @pytest.mark.skipif(yirgacheffe._backends.BACKEND != "NUMPY", reason="Only applies for numpy")
 def test_add_byte_layers_and_sum(monkeypatch) -> None:
     with monkeypatch.context() as m:

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/tests/test_pickle.py

@@ -30,10 +30,9 @@ def test_pickle_raster_layer() -> None:
 
 def test_pickle_raster_mem_layer_fails() -> None:
     area = Area(-10, 10, 10, -10)
-    layer = RasterLayer(gdal_dataset_of_region(area, 0.02))
-
-    with pytest.raises(ValueError):
-        _ = pickle.dumps(layer)
+    with RasterLayer(gdal_dataset_of_region(area, 0.02)) as layer:
+        with pytest.raises(ValueError):
+            _ = pickle.dumps(layer)
 
 def test_pickle_dyanamic_vector_layer() -> None:
     with tempfile.TemporaryDirectory() as tempdir:

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/tests/test_raster.py

@@ -91,6 +91,7 @@ def test_empty_layer_from_raster():
     empty = RasterLayer.empty_raster_layer_like(source)
     assert empty.pixel_scale == source.pixel_scale
     assert empty.projection == source.projection
+    assert empty.map_projection == source.map_projection
     assert empty.window == source.window
     assert empty.datatype == source.datatype
     assert empty.geo_transform == source.geo_transform
@@ -102,6 +103,7 @@ def test_empty_layer_from_raster_with_no_data_value(nodata):
     empty = RasterLayer.empty_raster_layer_like(source, nodata=nodata)
     assert empty.pixel_scale == source.pixel_scale
     assert empty.projection == source.projection
+    assert empty.map_projection == source.map_projection
     assert empty.window == source.window
     assert empty.datatype == source.datatype
     assert empty.geo_transform == source.geo_transform
@@ -113,6 +115,7 @@ def test_empty_layer_from_raster_with_new_smaller_area():
     empty = RasterLayer.empty_raster_layer_like(source, area=smaller_area)
     assert empty.pixel_scale == source.pixel_scale
     assert empty.projection == source.projection
+    assert empty.map_projection == source.map_projection
     assert empty.window == Window(0, 0, 100, 100)
     assert empty.datatype == source.datatype
     assert empty.geo_transform == (-1.0, 0.02, 0.0, 1.0, 0.0, -0.02)
@@ -123,6 +126,7 @@ def test_empty_layer_from_raster_new_datatype():
     empty = RasterLayer.empty_raster_layer_like(source, datatype=gdal.GDT_Float64)
     assert empty.pixel_scale == source.pixel_scale
     assert empty.projection == source.projection
+    assert empty.map_projection == source.map_projection
     assert empty.window == source.window
     assert empty.datatype == DataType.Float64
 
@@ -136,6 +140,7 @@ def test_empty_layer_from_raster_with_window():
     empty = RasterLayer.empty_raster_layer_like(source)
     assert empty.pixel_scale == source.pixel_scale
     assert empty.projection == source.projection
+    assert empty.map_projection == source.map_projection
     assert empty.window.xoff == 0
     assert empty.window.yoff == 0
     assert empty.window.xsize == source.window.xsize

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/tests/test_vectors.py

@@ -30,6 +30,7 @@ def test_basic_dynamic_vector_layer() -> None:
             assert layer.geo_transform == (area.left, 1.0, 0.0, area.top, 0.0, -1.0)
             assert layer.window == Window(0, 0, 20, 10)
             assert layer.projection == WGS_84_PROJECTION
+            assert layer.map_projection.name == WGS_84_PROJECTION
 
             # The astype here is to catch escaping MLX types...
             res = layer.read_array(0, 0, 20, 20).astype(int)
@@ -46,6 +47,7 @@ def test_rastered_vector_layer() -> None:
             assert layer.geo_transform == (area.left, 1.0, 0.0, area.top, 0.0, -1.0)
             assert layer.window == Window(0, 0, 20, 10)
             assert layer.projection == WGS_84_PROJECTION
+            assert layer.map_projection.name == WGS_84_PROJECTION
 
 def test_basic_dynamic_vector_layer_no_filter_match() -> None:
     with tempfile.TemporaryDirectory() as tempdir:
@@ -100,6 +102,7 @@ def test_empty_layer_from_vector():
         empty = RasterLayer.empty_raster_layer_like(source)
         assert empty.pixel_scale == source.pixel_scale
         assert empty.projection == source.projection
+        assert empty.map_projection == source.map_projection
         assert empty.window == source.window
         assert empty.area == source.area
 

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe/__init__.py

@@ -12,7 +12,7 @@ except ModuleNotFoundError:
         pyproject_data = tomllib.load(f)
     __version__ = pyproject_data["project"]["version"]
 
-from ._core import read_raster, read_rasters, read_shape, read_shape_like
+from ._core import read_raster, read_rasters, read_shape, read_shape_like, constant, read_narrow_raster
 from .constants import WGS_84_PROJECTION
 
 gdal.UseExceptions()

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe/_core.py

@@ -1,12 +1,14 @@
 from pathlib import Path
 from typing import Optional, Sequence, Tuple, Union
 
+from .layers.area import UniformAreaLayer
 from .layers.base import YirgacheffeLayer
+from .layers.constant import ConstantLayer
 from .layers.group import GroupLayer, TiledGroupLayer
 from .layers.rasters import RasterLayer
 from .layers.vectors import VectorLayer
 from .window import MapProjection
-from .operators import DataType
+from ._backends.enumeration import dtype as DataType
 
 def read_raster(
     filename: Union[Path,str],
@@ -31,6 +33,33 @@ def read_raster(
     """
     return RasterLayer.layer_from_file(filename, band, ignore_nodata)
 
+def read_narrow_raster(
+    filename: Union[Path,str],
+    band: int = 1,
+    ignore_nodata: bool = False,
+) -> RasterLayer:
+    """Open a 1 pixel wide raster file as a global raster.
+
+    This exists for the special use case where an area per pixel raster would have the same value per horizontal row
+    (e.g., a WGS84 map projection). For that case you can use this to load a raster that is 1 pixel wide and have
+    it automatically expanded to act like a global raster in calculations.
+
+    Parameters
+    ----------
+    filename : Path
+        Path of raster file to open.
+    band : int, default=1
+        For multi-band rasters, which band to use (defaults to first if not specified)
+    ignore_nodata : bool, default=False
+        If the GeoTIFF has a NODATA value, don't subsitute that value for NaN
+
+    Returns
+    -------
+    RasterLayer
+        Returns an layer representing the raster data.
+    """
+    return UniformAreaLayer.layer_from_file(filename, band, ignore_nodata)
+
 def read_rasters(
     filenames : Sequence[Union[Path,str]],
     tiled: bool=False
@@ -132,3 +161,23 @@ def read_shape_like(
         datatype,
         burn_value,
     )
+
+def constant(value: Union[int,float]) -> ConstantLayer:
+    """Generate a layer that has the same value in all pixels regardless of scale, projection, and area.
+
+    Generally this should not be necessary unless you must have the constant as the first term in an
+    expression, as Yirgacheffe will automatically convert numbers into constant layers. However if the
+    constant is the first term in the expression it must be wrapped by this call otherwise Python will
+    not know that it should be part of the Yirgacheffe expression.
+
+    Parameters
+    ----------
+    value : int or float
+        The value to be in each pixel of the expression term.
+
+    Returns
+    -------
+    ConstantLayer
+        Returns a constant layer of the provided value.
+    """
+    return ConstantLayer(value)

yirgacheffe-1.7.5/yirgacheffe/operators.py → yirgacheffe-1.7.7/yirgacheffe/_operators.py

@@ -2,10 +2,12 @@ import logging
 import math
 import multiprocessing
 import os
+import resource
 import sys
 import tempfile
 import time
 import types
+from contextlib import ExitStack
 from enum import Enum
 from multiprocessing import Semaphore, Process
 from multiprocessing.managers import SharedMemoryManager
@@ -387,50 +389,9 @@ class LayerOperation(LayerMathMixin):
 
     @property
     def area(self) -> Area:
-        # The type().__name__ here is to avoid a circular import dependancy
-        lhs_area = self.lhs.area
-        try:
-            rhs_area = self.rhs.area
-        except AttributeError:
-            rhs_area = None
-        try:
-            other_area = self.other.area
-        except AttributeError:
-            other_area = None
-
-        all_areas = [x for x in [lhs_area, rhs_area, other_area] if (x is not None) and (not x.is_world)]
-
-        match self.window_op:
-            case WindowOperation.NONE:
-                return all_areas[0]
-            case WindowOperation.LEFT:
-                return lhs_area
-            case WindowOperation.RIGHT:
-                assert rhs_area is not None
-                return rhs_area
-            case WindowOperation.INTERSECTION:
-                intersection = Area(
-                    left=max(x.left for x in all_areas),
-                    top=min(x.top for x in all_areas),
-                    right=min(x.right for x in all_areas),
-                    bottom=max(x.bottom for x in all_areas)
-                )
-                if (intersection.left >= intersection.right) or (intersection.bottom >= intersection.top):
-                    raise ValueError('No intersection possible')
-                return intersection
-            case WindowOperation.UNION:
-                return Area(
-                    left=min(x.left for x in all_areas),
-                    top=max(x.top for x in all_areas),
-                    right=max(x.right for x in all_areas),
-                    bottom=min(x.bottom for x in all_areas)
-                )
-            case _:
-                assert False, "Should not be reached"
+        return self._get_operation_area(self.map_projection)
 
     def _get_operation_area(self, projection: Optional[MapProjection]) -> Area:
-
-        # The type().__name__ here is to avoid a circular import dependancy
         lhs_area = self.lhs._get_operation_area(projection)
         try:
             rhs_area = self.rhs._get_operation_area(projection)
@@ -462,12 +423,13 @@ class LayerOperation(LayerMathMixin):
                     raise ValueError('No intersection possible')
                 return intersection
             case WindowOperation.UNION:
-                return Area(
+                union = Area(
                     left=min(x.left for x in all_areas),
                     top=max(x.top for x in all_areas),
                     right=max(x.right for x in all_areas),
                     bottom=min(x.bottom for x in all_areas)
                 )
+                return union
             case _:
                 assert False, "Should not be reached"
 
@@ -809,83 +771,92 @@ class LayerOperation(LayerMathMixin):
 
         total = 0.0
 
-        with multiprocessing.Manager() as manager:
-            with SharedMemoryManager() as smm:
-
-                mem_sem_cast = []
-                for _ in range(worker_count):
-                    shared_buf = smm.SharedMemory(size=np_dtype.itemsize * self.ystep * computation_window.xsize)
-                    cast_buf : npt.NDArray = np.ndarray(
-                        (self.ystep, computation_window.xsize),
-                        dtype=np_dtype,
-                        buffer=shared_buf.buf
-                    )
-                    cast_buf[:] = np.zeros((self.ystep, computation_window.xsize), np_dtype)
-                    mem_sem_cast.append((shared_buf, Semaphore(), cast_buf))
-
-                source_queue = manager.Queue()
-                result_queue = manager.Queue()
-
-                for yoffset in range(0, computation_window.ysize, self.ystep):
-                    step = ((computation_window.ysize - yoffset)
-                        if yoffset+self.ystep > computation_window.ysize
-                        else self.ystep)
-                    source_queue.put((
-                        yoffset,
-                        step
-                    ))
-                for _ in range(worker_count):
-                    source_queue.put(None)
-
-                if callback:
-                    callback(0.0)
-
-                workers = [Process(target=self._parallel_worker, args=(
-                    i,
-                    mem_sem_cast[i][0],
-                    mem_sem_cast[i][1],
-                    np_dtype,
-                    computation_window.xsize,
-                    source_queue,
-                    result_queue,
-                    computation_window
-                )) for i in range(worker_count)]
-                for worker in workers:
-                    worker.start()
-
-                sentinal_count = len(workers)
-                retired_blocks = 0
-                while sentinal_count > 0:
-                    res = result_queue.get()
-                    if res is None:
-                        sentinal_count -= 1
-                        continue
-                    index, yoffset, step = res
-                    _, sem, arr = mem_sem_cast[index]
-                    if band:
-                        band.WriteArray(
-                            arr[0:step],
-                            destination_window.xoff,
-                            yoffset + destination_window.yoff,
+        with ExitStack() as stack:
+            # If we get this far, then we're going to do the multiprocessing path. In general we've had
+            # a lot of issues with limits on open file descriptors using multiprocessing on bigger machines
+            # with hundreds of cores, and so to avoid blowing up in a way that is confusing to non-compsci
+            # types, we just set the soft ulimit as high as we can
+            previous_fd_limit, max_fd_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+            resource.setrlimit(resource.RLIMIT_NOFILE, (max_fd_limit, max_fd_limit))
+            stack.callback(resource.setrlimit, resource.RLIMIT_NOFILE, (previous_fd_limit, max_fd_limit))
+
+            with multiprocessing.Manager() as manager:
+                with SharedMemoryManager() as smm:
+
+                    mem_sem_cast = []
+                    for _ in range(worker_count):
+                        shared_buf = smm.SharedMemory(size=np_dtype.itemsize * self.ystep * computation_window.xsize)
+                        cast_buf : npt.NDArray = np.ndarray(
+                            (self.ystep, computation_window.xsize),
+                            dtype=np_dtype,
+                            buffer=shared_buf.buf
                         )
-                    if and_sum:
-                        total += np.sum(np.array(arr[0:step]).astype(np.float64))
-                    sem.release()
-                    retired_blocks += 1
+                        cast_buf[:] = np.zeros((self.ystep, computation_window.xsize), np_dtype)
+                        mem_sem_cast.append((shared_buf, Semaphore(), cast_buf))
+
+                    source_queue = manager.Queue()
+                    result_queue = manager.Queue()
+
+                    for yoffset in range(0, computation_window.ysize, self.ystep):
+                        step = ((computation_window.ysize - yoffset)
+                            if yoffset+self.ystep > computation_window.ysize
+                            else self.ystep)
+                        source_queue.put((
+                            yoffset,
+                            step
+                        ))
+                    for _ in range(worker_count):
+                        source_queue.put(None)
+
                     if callback:
-                        callback(retired_blocks / work_blocks)
-
-                processes = workers
-                while processes:
-                    candidates = [x for x in processes if not x.is_alive()]
-                    for candidate in candidates:
-                        candidate.join()
-                        if candidate.exitcode:
-                            for victim in processes:
-                                victim.kill()
-                            sys.exit(candidate.exitcode)
-                        processes.remove(candidate)
-                    time.sleep(0.01)
+                        callback(0.0)
+
+                    workers = [Process(target=self._parallel_worker, args=(
+                        i,
+                        mem_sem_cast[i][0],
+                        mem_sem_cast[i][1],
+                        np_dtype,
+                        computation_window.xsize,
+                        source_queue,
+                        result_queue,
+                        computation_window
+                    )) for i in range(worker_count)]
+                    for worker in workers:
+                        worker.start()
+
+                    sentinal_count = len(workers)
+                    retired_blocks = 0
+                    while sentinal_count > 0:
+                        res = result_queue.get()
+                        if res is None:
+                            sentinal_count -= 1
+                            continue
+                        index, yoffset, step = res
+                        _, sem, arr = mem_sem_cast[index]
+                        if band:
+                            band.WriteArray(
+                                arr[0:step],
+                                destination_window.xoff,
+                                yoffset + destination_window.yoff,
+                            )
+                        if and_sum:
+                            total += np.sum(np.array(arr[0:step]).astype(np.float64))
+                        sem.release()
+                        retired_blocks += 1
+                        if callback:
+                            callback(retired_blocks / work_blocks)
+
+                    processes = workers
+                    while processes:
+                        candidates = [x for x in processes if not x.is_alive()]
+                        for candidate in candidates:
+                            candidate.join()
+                            if candidate.exitcode:
+                                for victim in processes:
+                                    victim.kill()
+                                sys.exit(candidate.exitcode)
+                            processes.remove(candidate)
+                        time.sleep(0.01)
 
         return total if and_sum else None
 

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe/layers/base.py

@@ -4,10 +4,11 @@ from typing import Any, Optional, Sequence, Tuple
 import deprecation
 
 from .. import __version__
-from ..operators import DataType, LayerMathMixin
+from .._operators import LayerMathMixin
 from ..rounding import almost_equal, round_up_pixels, round_down_pixels
 from ..window import Area, MapProjection, PixelScale, Window
 from .._backends import backend
+from .._backends.enumeration import dtype as DataType
 
 class YirgacheffeLayer(LayerMathMixin):
     """The common base class for the different layer types. Most still inherit from RasterLayer as deep down

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe/layers/constant.py

@@ -1,10 +1,9 @@
 from typing import Any, Union
 
-from ..operators import DataType
 from ..window import Area, MapProjection, PixelScale, Window
 from .base import YirgacheffeLayer
 from .._backends import backend
-
+from .._backends.enumeration import dtype as DataType
 
 class ConstantLayer(YirgacheffeLayer):
     """This is a layer that will return the identity value - can be used when an input layer is

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe/layers/group.py

@@ -6,13 +6,12 @@ from typing import Any, List, Optional, Sequence, Union
 import numpy as np
 from numpy import ma
 
-from ..operators import DataType
 from ..rounding import round_down_pixels
 from ..window import Area, Window
 from .base import YirgacheffeLayer
 from .rasters import RasterLayer
 from .._backends import backend
-
+from .._backends.enumeration import dtype as DataType
 
 class GroupLayerEmpty(ValueError):
     def __init__(self, msg):

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe/layers/h3layer.py

@@ -3,12 +3,12 @@ from typing import Any, Tuple
 
 import h3
 import numpy as np
-from yirgacheffe.operators import DataType
 
 from ..rounding import round_up_pixels
 from ..window import Area, MapProjection, Window
 from .base import YirgacheffeLayer
 from .._backends import backend
+from .._backends.enumeration import dtype as DataType
 
 class H3CellLayer(YirgacheffeLayer):
 

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe/layers/rasters.py

@@ -10,8 +10,8 @@ from ..constants import WGS_84_PROJECTION
 from ..window import Area, MapProjection, PixelScale, Window
 from ..rounding import round_up_pixels
 from .base import YirgacheffeLayer
-from ..operators import DataType
 from .._backends import backend
+from .._backends.enumeration import dtype as DataType
 
 class InvalidRasterBand(Exception):
     def __init__ (self, band):
@@ -97,8 +97,6 @@ class RasterLayer(YirgacheffeLayer):
         threads: Optional[int]=None,
         bands: int=1
     ) -> RasterLayer:
-        width = layer.window.xsize
-        height = layer.window.ysize
         if area is None:
             area = layer.area
         assert area is not None
@@ -117,6 +115,12 @@ class RasterLayer(YirgacheffeLayer):
             area.left, projection.xstep, 0.0, area.top, 0.0, projection.ystep
         )
 
+        if area is None:
+            og_width = layer.window.xsize
+            og_height = layer.window.ysize
+            assert (og_width == width) and (og_height == height), \
+                f"original size ({og_width}, {og_height}) != estimated ({width}, {height})"
+
         if datatype is None:
             datatype_arg = layer.datatype
         elif isinstance(datatype, int):

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe/layers/rescaled.py

@@ -4,11 +4,11 @@ from pathlib import Path
 from typing import Any, Optional, Union
 
 from skimage import transform
-from yirgacheffe.operators import DataType
 
 from ..window import MapProjection, PixelScale, Window
 from .rasters import RasterLayer, YirgacheffeLayer
 from .._backends import backend
+from .._backends.enumeration import dtype as DataType
 
 
 class RescaledRasterLayer(YirgacheffeLayer):

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe/layers/vectors.py

@@ -4,13 +4,15 @@ from pathlib import Path
 from typing import Any, Optional, Tuple, Union
 from typing_extensions import NotRequired
 
+import deprecation
 from osgeo import gdal, ogr
 
-from ..operators import DataType
+from .. import __version__
 from ..window import Area, MapProjection, PixelScale
 from .base import YirgacheffeLayer
 from .rasters import RasterLayer
 from .._backends import backend
+from .._backends.enumeration import dtype as DataType
 
 def _validate_burn_value(burn_value: Any, layer: ogr.Layer) -> DataType: # pylint: disable=R0911
     if isinstance(burn_value, str):
@@ -60,6 +62,12 @@ class RasteredVectorLayer(RasterLayer):
     VectorLayer."""
 
     @classmethod
+    @deprecation.deprecated(
+        deprecated_in="1.7",
+        removed_in="2.0",
+        current_version=__version__,
+        details="Use `VectorLayer` instead."
+    )
     def layer_from_file( # type: ignore[override] # pylint: disable=W0221
         cls,
         filename: Union[Path,str],

yirgacheffe-1.7.7/yirgacheffe/operators.py

@@ -0,0 +1,7 @@
+# Eventually all this should be moved to the top level in 2.0, but for backwards compatibility in 1.x needs
+# to remain here
+
+from ._operators import where, minumum, maximum, clip, log, log2, log10, exp, exp2, nan_to_num, isin, \
+    floor, ceil # pylint: disable=W0611
+from ._operators import abs, round # pylint: disable=W0611,W0622
+from ._backends.enumeration import dtype as DataType # pylint: disable=W0611

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe/window.py

@@ -174,7 +174,7 @@ class Window:
         Y axis offset
     xsize : int
         Width of data in pixels
-    bottom : float
+    ysize : float
         Height of data in pixels
 
     Attributes
@@ -185,7 +185,7 @@ class Window:
         Y axis offset
     xsize : int
         Width of data in pixels
-    bottom : float
+    ysize : float
         Height of data in pixels
     """
     xoff: int

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7/yirgacheffe.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.7.5
+Version: 1.7.7
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC
@@ -262,17 +262,33 @@ Notes:
 * You can have missing tiles, and these will be filled in with zeros.
 * You can have tiles that overlap, so long as they still conform to the rule that all tiles are the same size and on a grid.
 
-### ConstantLayer
+### Constants
 
-This is there to simplify code when you have some optional layers. Rather than littering your code with checks, you can just use a constant layer, which can be included in calculations and will just return an fixed value as if it wasn't there. Useful with 0.0 or 1.0 for sum or multiplication null layers.
+At times it is useful to have a fixed constant in an expression. Typically, similar to numpy, if an expression involving layers has a constant in, Yirgacheffe will apply that to all pixels in the equation without need for further elaboration:
+
+```python
+with yg.read_raster("some_data.tif") as layer:
+    doubled_layer = layer * 2.0
+    ...
+```
+
+This can be useful in tasks where you have an optional layer in your code. For example, here the code optionally loads an area-per-pixel layer, which if not present can just be substituted with a 1.0:
 
 ```python
 try:
-    area_layer = RasterLayer.layer_from_file('myarea.tiff')
+    area_layer = yg.read_raster('myarea.tiff')
 except FileDoesNotExist:
-    area_layer = ConstantLayer(0.0)
+    area_layer = 1.0
 ```
 
+However, as with numpy, Python can not make the correct inference if the constant value is the first term in the equation. In that case you need to explicitly wrap the value with `constant` to help Python understand what is happening:
+
+```python
+with yg.read_raster("some_data.tif") as layer:
+    result = yg.constant(1.0) / layer
+```
+
+
 ### H3CellLayer
 
 If you have H3 installed, you can generate a mask layer based on an H3 cell identifier, where pixels inside the cell will have a value of 1, and those outside will have a value of 0.

{yirgacheffe-1.7.5 → yirgacheffe-1.7.7}/yirgacheffe.egg-info/SOURCES.txt

@@ -29,6 +29,7 @@ tests/test_vectors.py
 tests/test_window.py
 yirgacheffe/__init__.py
 yirgacheffe/_core.py
+yirgacheffe/_operators.py
 yirgacheffe/constants.py
 yirgacheffe/operators.py
 yirgacheffe/rounding.py