yirgacheffe 1.3.0__py3-none-any.whl → 1.3.2__py3-none-any.whl

yirgacheffe/constants.py

@@ -1 +1,2 @@
 YSTEP = 512
+MINIMUM_CHUNKS_PER_THREAD = 1

yirgacheffe/operators.py

@@ -1,4 +1,5 @@
 import logging
+import math
 import multiprocessing
 import sys
 import time
@@ -529,7 +530,7 @@ class LayerOperation(LayerMathMixin):
 
         return total if and_sum else None
 
-    def _parallel_worker(self, index, shared_mem, sem, np_dtype, width, input_queue, output_queue):
+    def _parallel_worker(self, index, shared_mem, sem, np_dtype, width, input_queue, output_queue, computation_window):
         arr = np.ndarray((self.ystep, width), dtype=np_dtype, buffer=shared_mem.buf)
 
         try:
@@ -549,7 +550,7 @@ class LayerOperation(LayerMathMixin):
                     break
                 yoffset, step = task
 
-                result = self._eval(self.area, yoffset, step)
+                result = self._eval(self.area, yoffset, step, computation_window)
                 backend.eval_op(result)
 
                 arr[:step] = backend.demote_array(result)
@@ -578,6 +579,20 @@ class LayerOperation(LayerMathMixin):
     def _parallel_save(self, destination_layer, and_sum=False, callback=None, parallelism=None, band=1):
         assert (destination_layer is not None) or and_sum
         computation_window = self.window
+
+        worker_count = parallelism or multiprocessing.cpu_count()
+        work_blocks = len(range(0, computation_window.ysize, self.ystep))
+        adjusted_blocks = math.ceil(work_blocks / constants.MINIMUM_CHUNKS_PER_THREAD)
+        worker_count = min(adjusted_blocks, worker_count)
+
+        if worker_count == 1:
+            if destination_layer:
+                return self.save(destination_layer, and_sum, callback, band)
+            elif and_sum:
+                return self.sum()
+            else:
+                assert False
+
         if destination_layer is not None:
             try:
                 band = destination_layer._dataset.GetRasterBand(band)
@@ -615,10 +630,6 @@ class LayerOperation(LayerMathMixin):
         with multiprocessing.Manager() as manager:
             with SharedMemoryManager() as smm:
 
-                worker_count = parallelism or multiprocessing.cpu_count()
-                work_blocks = len(range(0, computation_window.ysize, self.ystep))
-                worker_count = min(work_blocks, worker_count)
-
                 mem_sem_cast = []
                 for i in range(worker_count):
                     shared_buf = smm.SharedMemory(size=np_dtype.itemsize * self.ystep * computation_window.xsize)
@@ -650,7 +661,8 @@ class LayerOperation(LayerMathMixin):
                     np_dtype,
                     computation_window.xsize,
                     source_queue,
-                    result_queue
+                    result_queue,
+                    computation_window
                 )) for i in range(worker_count)]
                 for worker in workers:
                     worker.start()

{yirgacheffe-1.3.0.dist-info → yirgacheffe-1.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: yirgacheffe
-Version: 1.3.0
+Version: 1.3.2
 Summary: Abstraction of gdal datasets for doing basic math operations
 Author-email: Michael Dales <mwd24@cam.ac.uk>
 License-Expression: ISC

{yirgacheffe-1.3.0.dist-info → yirgacheffe-1.3.2.dist-info}/RECORD

@@ -1,7 +1,7 @@
 yirgacheffe/__init__.py,sha256=U5AoPk_iWreSCexG2ID-tmSXiJz4_9Lvzbf42DMvT7k,658
-yirgacheffe/constants.py,sha256=9ZcFLcir4GhPBBDG2rzLhDtJ0pMzpZYuzE9f-CFeA44,12
+yirgacheffe/constants.py,sha256=WccPcISG1FqL_Kw1tI72BJGjHy6RvEcGEx_I9RK776U,42
 yirgacheffe/h3layer.py,sha256=MT2hm6n64hzHSeRPvjn-CwErru937ntKXbEU7CIlPSU,91
-yirgacheffe/operators.py,sha256=dOzScup0rYKU2GbY3j5zFJJ0JMQomq6hN0c1nH1_igs,27198
+yirgacheffe/operators.py,sha256=f72xFsH6KmmWhrbsU0V7g_jgTJQ4BAaZzDhV7i-wPio,27607
 yirgacheffe/rounding.py,sha256=ggBG4lMyLMtHLW3dBxr3gBCcF2qhRrY5etZiFGlIoqA,2258
 yirgacheffe/window.py,sha256=0XZdwD4mz0bRU9eBhFY1Xk1hQt6FqCKp3BnUgxZup3c,5224
 yirgacheffe/backends/__init__.py,sha256=149-fg1PVXC36cgyuSZsU8SYOm65fzUmYN_MHZtEyrY,313
@@ -17,9 +17,9 @@ yirgacheffe/layers/h3layer.py,sha256=Cyrw_6nXc1a_Twsb0FfexKhHAbegKXStL3J5LLk2AX8
 yirgacheffe/layers/rasters.py,sha256=8mAbH49RtKIoA3h9mxWNbPzo8NJWDSfoRPT7fpkTJ4I,12328
 yirgacheffe/layers/rescaled.py,sha256=kWJlu7DuUB3nRDt3VtbJKiqBDQb2Ba8xzIIXTvOGdK8,2945
 yirgacheffe/layers/vectors.py,sha256=JOqM7Ym53ZdxWKpBJac-opPp2N_ynMfet1Xtsi85dpM,15087
-yirgacheffe-1.3.0.dist-info/licenses/LICENSE,sha256=dNSHwUCJr6axStTKDEdnJtfmDdFqlE3h1NPCveqPfnY,757
-yirgacheffe-1.3.0.dist-info/METADATA,sha256=jt79WCW-SqvTFkzqnHQC6VQoK0rJwo6R3uMePZD4CMc,20473
-yirgacheffe-1.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-yirgacheffe-1.3.0.dist-info/entry_points.txt,sha256=j4KgHXbVGbGyfTySc1ypBdERpfihO4WNjppvCdE9HjE,52
-yirgacheffe-1.3.0.dist-info/top_level.txt,sha256=9DBFlKO2Ld3hG6TuE3qOTd3Tt8ugTiXil4AN4Wr9_y0,12
-yirgacheffe-1.3.0.dist-info/RECORD,,
+yirgacheffe-1.3.2.dist-info/licenses/LICENSE,sha256=dNSHwUCJr6axStTKDEdnJtfmDdFqlE3h1NPCveqPfnY,757
+yirgacheffe-1.3.2.dist-info/METADATA,sha256=Nb9n3FKLxFJXTZZqrdWZAdfYbCogEMxWrrEEw96zE3M,20473
+yirgacheffe-1.3.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+yirgacheffe-1.3.2.dist-info/entry_points.txt,sha256=j4KgHXbVGbGyfTySc1ypBdERpfihO4WNjppvCdE9HjE,52
+yirgacheffe-1.3.2.dist-info/top_level.txt,sha256=9DBFlKO2Ld3hG6TuE3qOTd3Tt8ugTiXil4AN4Wr9_y0,12
+yirgacheffe-1.3.2.dist-info/RECORD,,