large-image-converter 1.29.7.dev20__py3-none-any.whl → 1.29.7.dev22__py3-none-any.whl

large_image_converter/__init__.py

@@ -109,7 +109,7 @@ def _data_from_large_image(path, outputPath, **kwargs):
         _pool_add(tasks, (pool.submit(
             _convert_via_vips, img, savePath, outputPath, mime=mime, forTiled=False), ))
         results['images'][key] = savePath
-    _drain_pool(pool, tasks)
+    _drain_pool(pool, tasks, 'associated images')
     return results
 
 
@@ -218,7 +218,7 @@ def _generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
                 _pool_add(tasks, (pool.submit(
                     _convert_via_vips, subInputPath, savePath, tempPath, False), ))
                 extraImages[key] = savePath
-    _drain_pool(pool, tasks)
+    _drain_pool(pool, tasks, 'subpage')
     _output_tiff(outputList, outputPath, tempPath, lidata, extraImages, **kwargs)
 
 
@@ -349,7 +349,7 @@ def _concurrency_to_value(_concurrency=None, **kwargs):
     return max(1, large_image.config.cpu_count(logical=True) + _concurrency)
 
 
-def _get_thread_pool(memoryLimit=None, **kwargs):
+def _get_thread_pool(memoryLimit=None, parentConcurrency=None, numItems=None, **kwargs):
     """
     Allocate a thread pool based on the specific concurrency.
 
@@ -357,12 +357,37 @@ def _get_thread_pool(memoryLimit=None, **kwargs):
         process per memoryLimit bytes of total memory.
     """
     concurrency = _concurrency_to_value(**kwargs)
+    if parentConcurrency and parentConcurrency > 1 and concurrency > 1:
+        concurrency = max(1, int(math.ceil(concurrency / parentConcurrency)))
     if memoryLimit:
+        if parentConcurrency:
+            memoryLimit *= parentConcurrency
         concurrency = min(concurrency, large_image.config.total_memory() // memoryLimit)
+    if numItems and numItems >= 1 and concurrency > numItems:
+        concurrency = numItems
     concurrency = max(1, concurrency)
     return concurrent.futures.ThreadPoolExecutor(max_workers=concurrency)
 
 
+def _pool_log(left, total, label):
+    """
+    Log processing within a pool.
+
+    :param left: units left to process.
+    :param total: total units left to process.
+    :param label: label to log describing what is being processed.
+    """
+    if not hasattr(logger, '_pool_log_starttime'):
+        logger._pool_log_starttime = time.time()
+    if not hasattr(logger, '_pool_log_lastlog'):
+        logger._pool_log_lastlog = time.time()
+    if time.time() - logger._pool_log_lastlog < 10:
+        return
+    elapsed = time.time() - logger._pool_log_starttime
+    logger.debug('%d/%d %s left %4.2fs', left, total, label, elapsed)
+    logger._pool_log_lastlog = time.time()
+
+
 def _pool_add(tasks, newtask):
     """
     Add a new task to a pool, then drain any finished tasks at the start of the
@@ -381,7 +406,7 @@ def _pool_add(tasks, newtask):
         tasks.pop(0)
 
 
-def _drain_pool(pool, tasks):
+def _drain_pool(pool, tasks, label=''):
     """
     Wait for all tasks in a pool to complete, then shutdown the pool.
 
@@ -389,6 +414,8 @@ def _drain_pool(pool, tasks):
     :param tasks: a list containing either lists or tuples, the last element
         of which is a task submitted to the pool.  Altered.
     """
+    numtasks = len(tasks)
+    _pool_log(len(tasks), numtasks, label)
     while len(tasks):
         # This allows better stopping on a SIGTERM
         try:
@@ -396,6 +423,7 @@ def _drain_pool(pool, tasks):
         except concurrent.futures.TimeoutError:
             continue
         tasks.pop(0)
+        _pool_log(len(tasks), numtasks, label)
     pool.shutdown(False)
 
 
@@ -507,7 +535,8 @@ def _convert_large_image_tile(tilelock, strips, tile):
         strips[ty] = strips[ty].insert(vimg, x, 0, expand=True)
 
 
-def _convert_large_image_frame(frame, numFrames, ts, frameOutputPath, tempPath, **kwargs):
+def _convert_large_image_frame(frame, numFrames, ts, frameOutputPath, tempPath,
+                               parentConcurrency=None, **kwargs):
     """
     Convert a single frame from a large_image source.  This parallelizes tile
     reads.  Once all tiles are converted to a composited vips image, a tiff
@@ -518,18 +547,24 @@ def _convert_large_image_frame(frame, numFrames, ts, frameOutputPath, tempPath,
     :param ts: the open tile source.
     :param frameOutputPath: the destination name for the tiff file.
     :param tempPath: a temporary file in a temporary directory.
+    :param parentConcurrency: amount of concurrency used by parent task.
     """
     # The iterator tile size is a balance between memory use and fewer calls
     # and file handles.
     _iterTileSize = 4096
     logger.info('Processing frame %d/%d', frame + 1, numFrames)
     strips = []
-    pool = _get_thread_pool(**kwargs)
+    pool = _get_thread_pool(
+        memoryLimit=FrameMemoryEstimate,
+        # allow multiple tiles even if we are using all the cores, as it
+        # balances I/O and computation
+        parentConcurrency=(parentConcurrency // 2),
+        **kwargs)
     tasks = []
     tilelock = threading.Lock()
     for tile in ts.tileIterator(tile_size=dict(width=_iterTileSize), frame=frame):
         _pool_add(tasks, (pool.submit(_convert_large_image_tile, tilelock, strips, tile), ))
-    _drain_pool(pool, tasks)
+    _drain_pool(pool, tasks, f'tiles from frame {frame + 1}/{numFrames}')
     minbands = min(strip.bands for strip in strips)
     maxbands = max(strip.bands for strip in strips)
     if minbands != maxbands:
@@ -556,20 +591,21 @@ def _convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs):
     numFrames = len(lidata['metadata'].get('frames', [0]))
     outputList = []
     tasks = []
-    pool = _get_thread_pool(memoryLimit=FrameMemoryEstimate, **kwargs)
     startFrame = 0
     endFrame = numFrames
     if kwargs.get('onlyFrame') is not None and str(kwargs.get('onlyFrame')):
         startFrame = int(kwargs.get('onlyFrame'))
         endFrame = startFrame + 1
+    pool = _get_thread_pool(memoryLimit=FrameMemoryEstimate,
+                            numItems=endFrame - startFrame, **kwargs)
     for frame in range(startFrame, endFrame):
         frameOutputPath = tempPath + '-%d-%s.tiff' % (
             frame + 1, time.strftime('%Y%m%d-%H%M%S'))
         _pool_add(tasks, (pool.submit(
             _convert_large_image_frame, frame, numFrames, ts, frameOutputPath,
-            tempPath, **kwargs), ))
+            tempPath, pool._max_workers, **kwargs), ))
         outputList.append(frameOutputPath)
-    _drain_pool(pool, tasks)
+    _drain_pool(pool, tasks, 'frames')
     _output_tiff(outputList, outputPath, tempPath, lidata, **kwargs)
 
 
@@ -891,6 +927,7 @@ def convert(inputPath, outputPath=None, **kwargs):  # noqa: C901
 
     :returns: outputPath if successful
     """
+    logger._pool_log_starttime = time.time()
     if kwargs.get('_concurrency'):
         os.environ['VIPS_CONCURRENCY'] = str(_concurrency_to_value(**kwargs))
     geospatial = kwargs.get('geospatial')

{large_image_converter-1.29.7.dev20.dist-info → large_image_converter-1.29.7.dev22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: large-image-converter
-Version: 1.29.7.dev20
+Version: 1.29.7.dev22
 Summary: Converter for Large Image.
 Author: Kitware Inc
 Author-email: kitware@kitware.com
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: large-image-source-tiff >=1.29.7.dev20
+Requires-Dist: large-image-source-tiff >=1.29.7.dev22
 Requires-Dist: numpy
 Requires-Dist: psutil
 Requires-Dist: pyvips
@@ -26,7 +26,7 @@ Requires-Dist: tifftools
 Provides-Extra: all
 Requires-Dist: glymur ; extra == 'all'
 Requires-Dist: gdal ; extra == 'all'
-Requires-Dist: large-image[sources] >=1.29.7.dev20 ; extra == 'all'
+Requires-Dist: large-image[sources] >=1.29.7.dev22 ; extra == 'all'
 Requires-Dist: packaging ; extra == 'all'
 Requires-Dist: scikit-image ; extra == 'all'
 Provides-Extra: geospatial
@@ -34,7 +34,7 @@ Requires-Dist: gdal ; extra == 'geospatial'
 Provides-Extra: jp2k
 Requires-Dist: glymur ; extra == 'jp2k'
 Provides-Extra: sources
-Requires-Dist: large-image[sources] >=1.29.7.dev20 ; extra == 'sources'
+Requires-Dist: large-image[sources] >=1.29.7.dev22 ; extra == 'sources'
 Provides-Extra: stats
 Requires-Dist: packaging ; extra == 'stats'
 Requires-Dist: scikit-image ; extra == 'stats'

large_image_converter-1.29.7.dev22.dist-info/RECORD

@@ -0,0 +1,9 @@
+large_image_converter/__init__.py,sha256=_ixZBvfS3K_jOh2T98o37EF8jjZLrpNTw-ZXkW9r--w,41181
+large_image_converter/__main__.py,sha256=bGQaCR01kTGTnW2c2Z9z5BIiPSpCRPfj7_kPgppLeB4,13353
+large_image_converter/format_aperio.py,sha256=NkKY3gaSjPlfxvcTQgFiMADdq_mkdpKjGQ7WT_QsvaM,11761
+large_image_converter-1.29.7.dev22.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
+large_image_converter-1.29.7.dev22.dist-info/METADATA,sha256=R1FzqzhSNe5RQoNZmGdG4uS_RKrwgTXMYNaWLhxqIvA,4646
+large_image_converter-1.29.7.dev22.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+large_image_converter-1.29.7.dev22.dist-info/entry_points.txt,sha256=TkXGWAaHgquk_ZXfqHTjQLjv8RZriv6ioiJd7JL6UIs,78
+large_image_converter-1.29.7.dev22.dist-info/top_level.txt,sha256=cdRa-ZWWKzVHFsjK1Aq1f4bzAMPc3sgCpprmvpYczKg,22
+large_image_converter-1.29.7.dev22.dist-info/RECORD,,

large_image_converter-1.29.7.dev20.dist-info/RECORD

@@ -1,9 +0,0 @@
-large_image_converter/__init__.py,sha256=tc_3t6OixaMdfSlwMMFLjQ3uuC8HZm7atoUbHU6UcHM,39472
-large_image_converter/__main__.py,sha256=bGQaCR01kTGTnW2c2Z9z5BIiPSpCRPfj7_kPgppLeB4,13353
-large_image_converter/format_aperio.py,sha256=NkKY3gaSjPlfxvcTQgFiMADdq_mkdpKjGQ7WT_QsvaM,11761
-large_image_converter-1.29.7.dev20.dist-info/LICENSE,sha256=psuoW8kuDP96RQsdhzwOqi6fyWv0ct8CR6Jr7He_P_k,10173
-large_image_converter-1.29.7.dev20.dist-info/METADATA,sha256=7T_cm4MpoP1B10gzPklBAtY-2Rtt-MKRrtEfHUhIT5w,4646
-large_image_converter-1.29.7.dev20.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-large_image_converter-1.29.7.dev20.dist-info/entry_points.txt,sha256=TkXGWAaHgquk_ZXfqHTjQLjv8RZriv6ioiJd7JL6UIs,78
-large_image_converter-1.29.7.dev20.dist-info/top_level.txt,sha256=cdRa-ZWWKzVHFsjK1Aq1f4bzAMPc3sgCpprmvpYczKg,22
-large_image_converter-1.29.7.dev20.dist-info/RECORD,,