yirgacheffe 1.7.6__tar.gz → 1.7.7__tar.gz

{yirgacheffe-1.7.6/yirgacheffe.egg-info → yirgacheffe-1.7.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.7.6
+Version: 1.7.7
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "yirgacheffe"
-version = "1.7.6"
+version = "1.7.7"
 description = "Abstraction of gdal datasets for doing basic math operations"
 readme = "README.md"
 authors = [{ name = "Michael Dales", email = "mwd24@cam.ac.uk" }]

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/tests/test_operators.py

@@ -10,7 +10,8 @@ import torch
 import yirgacheffe
 from yirgacheffe.window import Area, PixelScale
 from yirgacheffe.layers import ConstantLayer, RasterLayer, VectorLayer
-from yirgacheffe.operators import LayerOperation, DataType
+from yirgacheffe.operators import DataType
+from yirgacheffe._operators import LayerOperation
 from yirgacheffe._backends import backend
 from tests.helpers import gdal_dataset_with_data, gdal_dataset_of_region, make_vectors_with_id
 

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/tests/test_parallel_operators.py

@@ -1,4 +1,5 @@
 import os
+import resource
 import tempfile
 
 import numpy as np
@@ -8,7 +9,7 @@ import torch
 import yirgacheffe
 from tests.helpers import gdal_dataset_with_data
 from yirgacheffe.layers import RasterLayer
-from yirgacheffe.operators import LayerOperation
+from yirgacheffe._operators import LayerOperation
 
 # These tests are marked skip for MLX, because there seems to be a problem with
 # calling mx.eval in the tests for parallel save on Linux (which is what we use
@@ -42,7 +43,6 @@ def test_add_byte_layers_with_one_thread_uses_regular_save(monkeypatch) -> None:
             with pytest.raises(TypeError):
                 comp.parallel_save(result)
 
-
 @pytest.mark.skipif(yirgacheffe._backends.BACKEND != "NUMPY", reason="Only applies for numpy")
 def test_add_byte_layers(monkeypatch) -> None:
     with monkeypatch.context() as m:
@@ -71,6 +71,39 @@ def test_add_byte_layers(monkeypatch) -> None:
 
             assert (expected == actual).all()
 
+@pytest.mark.skipif(yirgacheffe._backends.BACKEND != "NUMPY", reason="Only applies for numpy")
+def test_rlimit_nofiles(monkeypatch) -> None:
+    with monkeypatch.context() as m:
+        m.setattr(yirgacheffe.constants, "YSTEP", 1)
+        m.setattr(LayerOperation, "save", None)
+        with tempfile.TemporaryDirectory() as tempdir:
+            path1 = os.path.join(tempdir, "test1.tif")
+            data1 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
+            dataset1 = gdal_dataset_with_data((0.0, 0.0), 0.02, data1, filename=path1)
+            dataset1.Close()
+
+            rlimit_log = []
+            def callback_rlimit_recorder(_progress: float) -> None:
+                rlimit_log.append(resource.getrlimit(resource.RLIMIT_NOFILE))
+
+            with RasterLayer.layer_from_file(path1) as layer:
+                with RasterLayer.empty_raster_layer_like(layer) as result:
+
+                    before_current_fd_limit, before_max_fd_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+
+                    comp = layer * 2
+                    comp.parallel_save(result, callback=callback_rlimit_recorder)
+
+                    after_current_fd_limit, after_max_fd_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+
+                    assert after_current_fd_limit == before_current_fd_limit
+                    assert after_max_fd_limit == before_max_fd_limit
+
+                    assert len(rlimit_log) > 0
+                    for recorded_current_fd_limit, recorded_max_fd_limit in rlimit_log:
+                        assert recorded_current_fd_limit == before_max_fd_limit
+                        assert recorded_max_fd_limit == before_max_fd_limit
+
 @pytest.mark.skipif(yirgacheffe._backends.BACKEND != "NUMPY", reason="Only applies for numpy")
 def test_add_byte_layers_and_sum(monkeypatch) -> None:
     with monkeypatch.context() as m:

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/yirgacheffe/_core.py

@@ -8,7 +8,7 @@ from .layers.group import GroupLayer, TiledGroupLayer
 from .layers.rasters import RasterLayer
 from .layers.vectors import VectorLayer
 from .window import MapProjection
-from .operators import DataType
+from ._backends.enumeration import dtype as DataType
 
 def read_raster(
     filename: Union[Path,str],

yirgacheffe-1.7.6/yirgacheffe/operators.py → yirgacheffe-1.7.7/yirgacheffe/_operators.py

@@ -2,10 +2,12 @@ import logging
 import math
 import multiprocessing
 import os
+import resource
 import sys
 import tempfile
 import time
 import types
+from contextlib import ExitStack
 from enum import Enum
 from multiprocessing import Semaphore, Process
 from multiprocessing.managers import SharedMemoryManager
@@ -769,83 +771,92 @@ class LayerOperation(LayerMathMixin):
 
         total = 0.0
 
-        with multiprocessing.Manager() as manager:
-            with SharedMemoryManager() as smm:
-
-                mem_sem_cast = []
-                for _ in range(worker_count):
-                    shared_buf = smm.SharedMemory(size=np_dtype.itemsize * self.ystep * computation_window.xsize)
-                    cast_buf : npt.NDArray = np.ndarray(
-                        (self.ystep, computation_window.xsize),
-                        dtype=np_dtype,
-                        buffer=shared_buf.buf
-                    )
-                    cast_buf[:] = np.zeros((self.ystep, computation_window.xsize), np_dtype)
-                    mem_sem_cast.append((shared_buf, Semaphore(), cast_buf))
-
-                source_queue = manager.Queue()
-                result_queue = manager.Queue()
-
-                for yoffset in range(0, computation_window.ysize, self.ystep):
-                    step = ((computation_window.ysize - yoffset)
-                        if yoffset+self.ystep > computation_window.ysize
-                        else self.ystep)
-                    source_queue.put((
-                        yoffset,
-                        step
-                    ))
-                for _ in range(worker_count):
-                    source_queue.put(None)
-
-                if callback:
-                    callback(0.0)
-
-                workers = [Process(target=self._parallel_worker, args=(
-                    i,
-                    mem_sem_cast[i][0],
-                    mem_sem_cast[i][1],
-                    np_dtype,
-                    computation_window.xsize,
-                    source_queue,
-                    result_queue,
-                    computation_window
-                )) for i in range(worker_count)]
-                for worker in workers:
-                    worker.start()
-
-                sentinal_count = len(workers)
-                retired_blocks = 0
-                while sentinal_count > 0:
-                    res = result_queue.get()
-                    if res is None:
-                        sentinal_count -= 1
-                        continue
-                    index, yoffset, step = res
-                    _, sem, arr = mem_sem_cast[index]
-                    if band:
-                        band.WriteArray(
-                            arr[0:step],
-                            destination_window.xoff,
-                            yoffset + destination_window.yoff,
+        with ExitStack() as stack:
+            # If we get this far, then we're going to do the multiprocessing path. In general we've had
+            # a lot of issues with limits on open file descriptors using multiprocessing on bigger machines
+            # with hundreds of cores, and so to avoid blowing up in a way that is confusing to non-compsci
+            # types, we just set the soft ulimit as high as we can
+            previous_fd_limit, max_fd_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+            resource.setrlimit(resource.RLIMIT_NOFILE, (max_fd_limit, max_fd_limit))
+            stack.callback(resource.setrlimit, resource.RLIMIT_NOFILE, (previous_fd_limit, max_fd_limit))
+
+            with multiprocessing.Manager() as manager:
+                with SharedMemoryManager() as smm:
+
+                    mem_sem_cast = []
+                    for _ in range(worker_count):
+                        shared_buf = smm.SharedMemory(size=np_dtype.itemsize * self.ystep * computation_window.xsize)
+                        cast_buf : npt.NDArray = np.ndarray(
+                            (self.ystep, computation_window.xsize),
+                            dtype=np_dtype,
+                            buffer=shared_buf.buf
                         )
-                    if and_sum:
-                        total += np.sum(np.array(arr[0:step]).astype(np.float64))
-                    sem.release()
-                    retired_blocks += 1
+                        cast_buf[:] = np.zeros((self.ystep, computation_window.xsize), np_dtype)
+                        mem_sem_cast.append((shared_buf, Semaphore(), cast_buf))
+
+                    source_queue = manager.Queue()
+                    result_queue = manager.Queue()
+
+                    for yoffset in range(0, computation_window.ysize, self.ystep):
+                        step = ((computation_window.ysize - yoffset)
+                            if yoffset+self.ystep > computation_window.ysize
+                            else self.ystep)
+                        source_queue.put((
+                            yoffset,
+                            step
+                        ))
+                    for _ in range(worker_count):
+                        source_queue.put(None)
+
                     if callback:
-                        callback(retired_blocks / work_blocks)
-
-                processes = workers
-                while processes:
-                    candidates = [x for x in processes if not x.is_alive()]
-                    for candidate in candidates:
-                        candidate.join()
-                        if candidate.exitcode:
-                            for victim in processes:
-                                victim.kill()
-                            sys.exit(candidate.exitcode)
-                        processes.remove(candidate)
-                    time.sleep(0.01)
+                        callback(0.0)
+
+                    workers = [Process(target=self._parallel_worker, args=(
+                        i,
+                        mem_sem_cast[i][0],
+                        mem_sem_cast[i][1],
+                        np_dtype,
+                        computation_window.xsize,
+                        source_queue,
+                        result_queue,
+                        computation_window
+                    )) for i in range(worker_count)]
+                    for worker in workers:
+                        worker.start()
+
+                    sentinal_count = len(workers)
+                    retired_blocks = 0
+                    while sentinal_count > 0:
+                        res = result_queue.get()
+                        if res is None:
+                            sentinal_count -= 1
+                            continue
+                        index, yoffset, step = res
+                        _, sem, arr = mem_sem_cast[index]
+                        if band:
+                            band.WriteArray(
+                                arr[0:step],
+                                destination_window.xoff,
+                                yoffset + destination_window.yoff,
+                            )
+                        if and_sum:
+                            total += np.sum(np.array(arr[0:step]).astype(np.float64))
+                        sem.release()
+                        retired_blocks += 1
+                        if callback:
+                            callback(retired_blocks / work_blocks)
+
+                    processes = workers
+                    while processes:
+                        candidates = [x for x in processes if not x.is_alive()]
+                        for candidate in candidates:
+                            candidate.join()
+                            if candidate.exitcode:
+                                for victim in processes:
+                                    victim.kill()
+                                sys.exit(candidate.exitcode)
+                            processes.remove(candidate)
+                        time.sleep(0.01)
 
         return total if and_sum else None
 

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/yirgacheffe/layers/base.py

@@ -4,10 +4,11 @@ from typing import Any, Optional, Sequence, Tuple
 import deprecation
 
 from .. import __version__
-from ..operators import DataType, LayerMathMixin
+from .._operators import LayerMathMixin
 from ..rounding import almost_equal, round_up_pixels, round_down_pixels
 from ..window import Area, MapProjection, PixelScale, Window
 from .._backends import backend
+from .._backends.enumeration import dtype as DataType
 
 class YirgacheffeLayer(LayerMathMixin):
     """The common base class for the different layer types. Most still inherit from RasterLayer as deep down

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/yirgacheffe/layers/constant.py

@@ -1,10 +1,9 @@
 from typing import Any, Union
 
-from ..operators import DataType
 from ..window import Area, MapProjection, PixelScale, Window
 from .base import YirgacheffeLayer
 from .._backends import backend
-
+from .._backends.enumeration import dtype as DataType
 
 class ConstantLayer(YirgacheffeLayer):
     """This is a layer that will return the identity value - can be used when an input layer is

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/yirgacheffe/layers/group.py

@@ -6,13 +6,12 @@ from typing import Any, List, Optional, Sequence, Union
 import numpy as np
 from numpy import ma
 
-from ..operators import DataType
 from ..rounding import round_down_pixels
 from ..window import Area, Window
 from .base import YirgacheffeLayer
 from .rasters import RasterLayer
 from .._backends import backend
-
+from .._backends.enumeration import dtype as DataType
 
 class GroupLayerEmpty(ValueError):
     def __init__(self, msg):

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/yirgacheffe/layers/h3layer.py

@@ -3,12 +3,12 @@ from typing import Any, Tuple
 
 import h3
 import numpy as np
-from yirgacheffe.operators import DataType
 
 from ..rounding import round_up_pixels
 from ..window import Area, MapProjection, Window
 from .base import YirgacheffeLayer
 from .._backends import backend
+from .._backends.enumeration import dtype as DataType
 
 class H3CellLayer(YirgacheffeLayer):
 

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/yirgacheffe/layers/rasters.py

@@ -10,8 +10,8 @@ from ..constants import WGS_84_PROJECTION
 from ..window import Area, MapProjection, PixelScale, Window
 from ..rounding import round_up_pixels
 from .base import YirgacheffeLayer
-from ..operators import DataType
 from .._backends import backend
+from .._backends.enumeration import dtype as DataType
 
 class InvalidRasterBand(Exception):
     def __init__ (self, band):

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/yirgacheffe/layers/rescaled.py

@@ -4,11 +4,11 @@ from pathlib import Path
 from typing import Any, Optional, Union
 
 from skimage import transform
-from yirgacheffe.operators import DataType
 
 from ..window import MapProjection, PixelScale, Window
 from .rasters import RasterLayer, YirgacheffeLayer
 from .._backends import backend
+from .._backends.enumeration import dtype as DataType
 
 
 class RescaledRasterLayer(YirgacheffeLayer):

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/yirgacheffe/layers/vectors.py

@@ -8,11 +8,11 @@ import deprecation
 from osgeo import gdal, ogr
 
 from .. import __version__
-from ..operators import DataType
 from ..window import Area, MapProjection, PixelScale
 from .base import YirgacheffeLayer
 from .rasters import RasterLayer
 from .._backends import backend
+from .._backends.enumeration import dtype as DataType
 
 def _validate_burn_value(burn_value: Any, layer: ogr.Layer) -> DataType: # pylint: disable=R0911
     if isinstance(burn_value, str):

yirgacheffe-1.7.7/yirgacheffe/operators.py

@@ -0,0 +1,7 @@
+# Eventually all this should be moved to the top level in 2.0, but for backwards compatibility in 1.x needs
+# to remain here
+
+from ._operators import where, minumum, maximum, clip, log, log2, log10, exp, exp2, nan_to_num, isin, \
+    floor, ceil # pylint: disable=W0611
+from ._operators import abs, round # pylint: disable=W0611,W0622
+from ._backends.enumeration import dtype as DataType # pylint: disable=W0611

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/yirgacheffe/window.py

@@ -174,7 +174,7 @@ class Window:
         Y axis offset
     xsize : int
         Width of data in pixels
-    bottom : float
+    ysize : float
         Height of data in pixels
 
     Attributes
@@ -185,7 +185,7 @@ class Window:
         Y axis offset
     xsize : int
         Width of data in pixels
-    bottom : float
+    ysize : float
         Height of data in pixels
     """
     xoff: int

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7/yirgacheffe.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.7.6
+Version: 1.7.7
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC

{yirgacheffe-1.7.6 → yirgacheffe-1.7.7}/yirgacheffe.egg-info/SOURCES.txt

@@ -29,6 +29,7 @@ tests/test_vectors.py
 tests/test_window.py
 yirgacheffe/__init__.py
 yirgacheffe/_core.py
+yirgacheffe/_operators.py
 yirgacheffe/constants.py
 yirgacheffe/operators.py
 yirgacheffe/rounding.py