large-image-converter 1.26.3.dev10__tar.gz → 1.33.4.dev39__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/PKG-INFO +35 -7
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter/__init__.py +176 -41
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter/__main__.py +2 -2
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter.egg-info/PKG-INFO +35 -7
- large_image_converter-1.33.4.dev39/large_image_converter.egg-info/requires.txt +25 -0
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/pyproject.toml +4 -0
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/setup.py +15 -25
- large-image-converter-1.26.3.dev10/large_image_converter.egg-info/requires.txt +0 -16
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/LICENSE +0 -0
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/README.rst +0 -0
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter/format_aperio.py +0 -0
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter.egg-info/SOURCES.txt +0 -0
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter.egg-info/dependency_links.txt +0 -0
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter.egg-info/entry_points.txt +0 -0
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter.egg-info/top_level.txt +0 -0
- {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/setup.cfg +0 -0
|
@@ -1,25 +1,53 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: large-image-converter
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.33.4.dev39
|
|
4
4
|
Summary: Converter for Large Image.
|
|
5
5
|
Author: Kitware Inc
|
|
6
6
|
Author-email: kitware@kitware.com
|
|
7
|
-
License: Apache
|
|
7
|
+
License: Apache-2.0
|
|
8
8
|
Classifier: Development Status :: 5 - Production/Stable
|
|
9
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
9
|
Classifier: Topic :: Scientific/Engineering
|
|
11
10
|
Classifier: Intended Audience :: Science/Research
|
|
12
11
|
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
14
12
|
Classifier: Programming Language :: Python :: 3.9
|
|
15
13
|
Classifier: Programming Language :: Python :: 3.10
|
|
16
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
17
15
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/x-rst
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: large-image-source-tiff>=1.33.4.dev39
|
|
21
|
+
Requires-Dist: numpy
|
|
22
|
+
Requires-Dist: psutil
|
|
23
|
+
Requires-Dist: pyvips
|
|
24
|
+
Requires-Dist: tifftools
|
|
19
25
|
Provides-Extra: jp2k
|
|
26
|
+
Requires-Dist: glymur; extra == "jp2k"
|
|
27
|
+
Provides-Extra: geospatial
|
|
28
|
+
Requires-Dist: gdal; extra == "geospatial"
|
|
20
29
|
Provides-Extra: sources
|
|
30
|
+
Requires-Dist: large-image[sources]>=1.33.4.dev39; extra == "sources"
|
|
21
31
|
Provides-Extra: stats
|
|
22
|
-
|
|
32
|
+
Requires-Dist: packaging; extra == "stats"
|
|
33
|
+
Requires-Dist: scikit-image; extra == "stats"
|
|
34
|
+
Provides-Extra: all
|
|
35
|
+
Requires-Dist: glymur; extra == "all"
|
|
36
|
+
Requires-Dist: gdal; extra == "all"
|
|
37
|
+
Requires-Dist: large-image[sources]>=1.33.4.dev39; extra == "all"
|
|
38
|
+
Requires-Dist: packaging; extra == "all"
|
|
39
|
+
Requires-Dist: scikit-image; extra == "all"
|
|
40
|
+
Dynamic: author
|
|
41
|
+
Dynamic: author-email
|
|
42
|
+
Dynamic: classifier
|
|
43
|
+
Dynamic: description
|
|
44
|
+
Dynamic: description-content-type
|
|
45
|
+
Dynamic: license
|
|
46
|
+
Dynamic: license-file
|
|
47
|
+
Dynamic: provides-extra
|
|
48
|
+
Dynamic: requires-dist
|
|
49
|
+
Dynamic: requires-python
|
|
50
|
+
Dynamic: summary
|
|
23
51
|
|
|
24
52
|
*********************
|
|
25
53
|
Large Image Converter
|
|
@@ -14,10 +14,12 @@ from importlib.metadata import version as _importlib_version
|
|
|
14
14
|
from tempfile import TemporaryDirectory
|
|
15
15
|
|
|
16
16
|
import numpy as np
|
|
17
|
-
import psutil
|
|
18
17
|
import tifftools
|
|
19
18
|
|
|
20
19
|
import large_image
|
|
20
|
+
from large_image.tilesource.utilities import (_gdalParameters,
|
|
21
|
+
_newFromFileLock, _vipsCast,
|
|
22
|
+
_vipsParameters)
|
|
21
23
|
|
|
22
24
|
from . import format_aperio
|
|
23
25
|
|
|
@@ -76,9 +78,9 @@ def _data_from_large_image(path, outputPath, **kwargs):
|
|
|
76
78
|
_import_pyvips()
|
|
77
79
|
if not path.startswith('large_image://test'):
|
|
78
80
|
try:
|
|
79
|
-
ts = large_image.open(path)
|
|
81
|
+
ts = large_image.open(path, noCache=True)
|
|
80
82
|
except Exception:
|
|
81
|
-
return
|
|
83
|
+
return None
|
|
82
84
|
else:
|
|
83
85
|
import urllib.parse
|
|
84
86
|
|
|
@@ -107,7 +109,7 @@ def _data_from_large_image(path, outputPath, **kwargs):
|
|
|
107
109
|
_pool_add(tasks, (pool.submit(
|
|
108
110
|
_convert_via_vips, img, savePath, outputPath, mime=mime, forTiled=False), ))
|
|
109
111
|
results['images'][key] = savePath
|
|
110
|
-
_drain_pool(pool, tasks)
|
|
112
|
+
_drain_pool(pool, tasks, 'associated images')
|
|
111
113
|
return results
|
|
112
114
|
|
|
113
115
|
|
|
@@ -129,7 +131,7 @@ def _generate_geotiff(inputPath, outputPath, **kwargs):
|
|
|
129
131
|
"""
|
|
130
132
|
from osgeo import gdal, gdalconst
|
|
131
133
|
|
|
132
|
-
cmdopt =
|
|
134
|
+
cmdopt = _gdalParameters(**kwargs)
|
|
133
135
|
cmd = ['gdal_translate', inputPath, outputPath] + cmdopt
|
|
134
136
|
logger.info('Convert to geotiff: %r', cmd)
|
|
135
137
|
try:
|
|
@@ -163,7 +165,8 @@ def _generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
|
|
|
163
165
|
"""
|
|
164
166
|
_import_pyvips()
|
|
165
167
|
|
|
166
|
-
|
|
168
|
+
with _newFromFileLock:
|
|
169
|
+
image = pyvips.Image.new_from_file(inputPath)
|
|
167
170
|
width = image.width
|
|
168
171
|
height = image.height
|
|
169
172
|
pages = 1
|
|
@@ -179,7 +182,8 @@ def _generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
|
|
|
179
182
|
# Process each image separately to pyramidize it
|
|
180
183
|
for page in range(pages):
|
|
181
184
|
subInputPath = inputPath + '[page=%d]' % page
|
|
182
|
-
|
|
185
|
+
with _newFromFileLock:
|
|
186
|
+
subImage = pyvips.Image.new_from_file(subInputPath)
|
|
183
187
|
imageSizes.append((subImage.width, subImage.height, subInputPath, page))
|
|
184
188
|
if subImage.width != width or subImage.height != height:
|
|
185
189
|
if subImage.width * subImage.height <= width * height:
|
|
@@ -214,7 +218,7 @@ def _generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
|
|
|
214
218
|
_pool_add(tasks, (pool.submit(
|
|
215
219
|
_convert_via_vips, subInputPath, savePath, tempPath, False), ))
|
|
216
220
|
extraImages[key] = savePath
|
|
217
|
-
_drain_pool(pool, tasks)
|
|
221
|
+
_drain_pool(pool, tasks, 'subpage')
|
|
218
222
|
_output_tiff(outputList, outputPath, tempPath, lidata, extraImages, **kwargs)
|
|
219
223
|
|
|
220
224
|
|
|
@@ -245,7 +249,7 @@ def _generate_tiff(inputPath, outputPath, tempPath, lidata, **kwargs):
|
|
|
245
249
|
|
|
246
250
|
|
|
247
251
|
def _convert_via_vips(inputPathOrBuffer, outputPath, tempPath, forTiled=True,
|
|
248
|
-
status=None, **kwargs):
|
|
252
|
+
status=None, preferredVipsCast=None, **kwargs):
|
|
249
253
|
"""
|
|
250
254
|
Convert a file, buffer, or vips image to a tiff file. This is equivalent
|
|
251
255
|
to a vips command line of
|
|
@@ -259,11 +263,12 @@ def _convert_via_vips(inputPathOrBuffer, outputPath, tempPath, forTiled=True,
|
|
|
259
263
|
also stores files in TMPDIR
|
|
260
264
|
:param forTiled: True if the output should be tiled, false if not.
|
|
261
265
|
:param status: an optional additional string to add to log messages.
|
|
266
|
+
:param preferredVipsCast: vips scaling parameters to use in a cast.
|
|
262
267
|
:param kwargs: addition arguments that get passed to _vipsParameters
|
|
263
268
|
and _convert_to_jp2k.
|
|
264
269
|
"""
|
|
265
270
|
_import_pyvips()
|
|
266
|
-
convertParams =
|
|
271
|
+
convertParams = _vipsParameters(forTiled, **kwargs)
|
|
267
272
|
status = (', ' + status) if status else ''
|
|
268
273
|
if isinstance(inputPathOrBuffer, pyvips.vimage.Image):
|
|
269
274
|
source = 'vips image'
|
|
@@ -273,23 +278,25 @@ def _convert_via_vips(inputPathOrBuffer, outputPath, tempPath, forTiled=True,
|
|
|
273
278
|
image = pyvips.Image.new_from_buffer(inputPathOrBuffer, '')
|
|
274
279
|
else:
|
|
275
280
|
source = inputPathOrBuffer
|
|
276
|
-
|
|
281
|
+
with _newFromFileLock:
|
|
282
|
+
image = pyvips.Image.new_from_file(inputPathOrBuffer)
|
|
277
283
|
logger.info('Input: %s, Output: %s, Options: %r%s',
|
|
278
284
|
source, outputPath, convertParams, status)
|
|
279
285
|
image = image.autorot()
|
|
280
286
|
adjusted = format_hook('modify_vips_image_before_output', image, convertParams, **kwargs)
|
|
281
287
|
if adjusted is False:
|
|
282
288
|
return
|
|
283
|
-
|
|
289
|
+
if adjusted:
|
|
284
290
|
image = adjusted
|
|
285
291
|
if (convertParams['compression'] not in {'jpeg'} or
|
|
286
292
|
image.interpretation != pyvips.Interpretation.SCRGB):
|
|
287
293
|
# jp2k compression supports more than 8-bits per sample, but the
|
|
288
294
|
# decompressor claims this is unsupported.
|
|
289
|
-
image =
|
|
295
|
+
image = _vipsCast(
|
|
290
296
|
image,
|
|
291
297
|
convertParams['compression'] in {'webp', 'jpeg'} or
|
|
292
|
-
kwargs.get('compression') in {'jp2k'}
|
|
298
|
+
kwargs.get('compression') in {'jp2k'},
|
|
299
|
+
preferredVipsCast)
|
|
293
300
|
# TODO: revisit the TMPDIR override; this is not thread safe
|
|
294
301
|
# oldtmpdir = os.environ.get('TMPDIR')
|
|
295
302
|
# os.environ['TMPDIR'] = os.path.dirname(tempPath)
|
|
@@ -341,10 +348,10 @@ def _concurrency_to_value(_concurrency=None, **kwargs):
|
|
|
341
348
|
_concurrency = int(_concurrency) if str(_concurrency).isdigit() else 0
|
|
342
349
|
if _concurrency > 0:
|
|
343
350
|
return _concurrency
|
|
344
|
-
return max(1,
|
|
351
|
+
return max(1, large_image.config.cpu_count(logical=True) + _concurrency)
|
|
345
352
|
|
|
346
353
|
|
|
347
|
-
def _get_thread_pool(memoryLimit=None, **kwargs):
|
|
354
|
+
def _get_thread_pool(memoryLimit=None, parentConcurrency=None, numItems=None, **kwargs):
|
|
348
355
|
"""
|
|
349
356
|
Allocate a thread pool based on the specific concurrency.
|
|
350
357
|
|
|
@@ -352,12 +359,37 @@ def _get_thread_pool(memoryLimit=None, **kwargs):
|
|
|
352
359
|
process per memoryLimit bytes of total memory.
|
|
353
360
|
"""
|
|
354
361
|
concurrency = _concurrency_to_value(**kwargs)
|
|
362
|
+
if parentConcurrency and parentConcurrency > 1 and concurrency > 1:
|
|
363
|
+
concurrency = max(1, int(math.ceil(concurrency / parentConcurrency)))
|
|
355
364
|
if memoryLimit:
|
|
356
|
-
|
|
365
|
+
if parentConcurrency:
|
|
366
|
+
memoryLimit *= parentConcurrency
|
|
367
|
+
concurrency = min(concurrency, large_image.config.total_memory() // memoryLimit)
|
|
368
|
+
if numItems and numItems >= 1 and concurrency > numItems:
|
|
369
|
+
concurrency = numItems
|
|
357
370
|
concurrency = max(1, concurrency)
|
|
358
371
|
return concurrent.futures.ThreadPoolExecutor(max_workers=concurrency)
|
|
359
372
|
|
|
360
373
|
|
|
374
|
+
def _pool_log(left, total, label):
|
|
375
|
+
"""
|
|
376
|
+
Log processing within a pool.
|
|
377
|
+
|
|
378
|
+
:param left: units left to process.
|
|
379
|
+
:param total: total units left to process.
|
|
380
|
+
:param label: label to log describing what is being processed.
|
|
381
|
+
"""
|
|
382
|
+
if not hasattr(logger, '_pool_log_starttime'):
|
|
383
|
+
logger._pool_log_starttime = time.time()
|
|
384
|
+
if not hasattr(logger, '_pool_log_lastlog'):
|
|
385
|
+
logger._pool_log_lastlog = time.time()
|
|
386
|
+
if time.time() - logger._pool_log_lastlog < 10:
|
|
387
|
+
return
|
|
388
|
+
elapsed = time.time() - logger._pool_log_starttime
|
|
389
|
+
logger.debug('%d/%d %s left %4.2fs', left, total, label, elapsed)
|
|
390
|
+
logger._pool_log_lastlog = time.time()
|
|
391
|
+
|
|
392
|
+
|
|
361
393
|
def _pool_add(tasks, newtask):
|
|
362
394
|
"""
|
|
363
395
|
Add a new task to a pool, then drain any finished tasks at the start of the
|
|
@@ -376,7 +408,7 @@ def _pool_add(tasks, newtask):
|
|
|
376
408
|
tasks.pop(0)
|
|
377
409
|
|
|
378
410
|
|
|
379
|
-
def _drain_pool(pool, tasks):
|
|
411
|
+
def _drain_pool(pool, tasks, label=''):
|
|
380
412
|
"""
|
|
381
413
|
Wait for all tasks in a pool to complete, then shutdown the pool.
|
|
382
414
|
|
|
@@ -384,6 +416,8 @@ def _drain_pool(pool, tasks):
|
|
|
384
416
|
:param tasks: a list containing either lists or tuples, the last element
|
|
385
417
|
of which is a task submitted to the pool. Altered.
|
|
386
418
|
"""
|
|
419
|
+
numtasks = len(tasks)
|
|
420
|
+
_pool_log(len(tasks), numtasks, label)
|
|
387
421
|
while len(tasks):
|
|
388
422
|
# This allows better stopping on a SIGTERM
|
|
389
423
|
try:
|
|
@@ -391,6 +425,7 @@ def _drain_pool(pool, tasks):
|
|
|
391
425
|
except concurrent.futures.TimeoutError:
|
|
392
426
|
continue
|
|
393
427
|
tasks.pop(0)
|
|
428
|
+
_pool_log(len(tasks), numtasks, label)
|
|
394
429
|
pool.shutdown(False)
|
|
395
430
|
|
|
396
431
|
|
|
@@ -502,7 +537,9 @@ def _convert_large_image_tile(tilelock, strips, tile):
|
|
|
502
537
|
strips[ty] = strips[ty].insert(vimg, x, 0, expand=True)
|
|
503
538
|
|
|
504
539
|
|
|
505
|
-
def _convert_large_image_frame(
|
|
540
|
+
def _convert_large_image_frame(
|
|
541
|
+
frame, numFrames, ts, frameOutputPath, tempPath, preferredVipsCast=None,
|
|
542
|
+
parentConcurrency=None, **kwargs):
|
|
506
543
|
"""
|
|
507
544
|
Convert a single frame from a large_image source. This parallelizes tile
|
|
508
545
|
reads. Once all tiles are converted to a composited vips image, a tiff
|
|
@@ -513,23 +550,103 @@ def _convert_large_image_frame(frame, numFrames, ts, frameOutputPath, tempPath,
|
|
|
513
550
|
:param ts: the open tile source.
|
|
514
551
|
:param frameOutputPath: the destination name for the tiff file.
|
|
515
552
|
:param tempPath: a temporary file in a temporary directory.
|
|
553
|
+
:param preferredVipsCast: vips scaling parameters to use in a cast.
|
|
554
|
+
:param parentConcurrency: amount of concurrency used by parent task.
|
|
516
555
|
"""
|
|
517
556
|
# The iterator tile size is a balance between memory use and fewer calls
|
|
518
557
|
# and file handles.
|
|
519
558
|
_iterTileSize = 4096
|
|
520
559
|
logger.info('Processing frame %d/%d', frame + 1, numFrames)
|
|
521
560
|
strips = []
|
|
522
|
-
pool = _get_thread_pool(
|
|
561
|
+
pool = _get_thread_pool(
|
|
562
|
+
memoryLimit=FrameMemoryEstimate,
|
|
563
|
+
# allow multiple tiles even if we are using all the cores, as it
|
|
564
|
+
# balances I/O and computation
|
|
565
|
+
parentConcurrency=(parentConcurrency // 2),
|
|
566
|
+
**kwargs)
|
|
523
567
|
tasks = []
|
|
524
568
|
tilelock = threading.Lock()
|
|
525
569
|
for tile in ts.tileIterator(tile_size=dict(width=_iterTileSize), frame=frame):
|
|
526
570
|
_pool_add(tasks, (pool.submit(_convert_large_image_tile, tilelock, strips, tile), ))
|
|
527
|
-
_drain_pool(pool, tasks)
|
|
571
|
+
_drain_pool(pool, tasks, f'tiles from frame {frame + 1}/{numFrames}')
|
|
572
|
+
minbands = min(strip.bands for strip in strips)
|
|
573
|
+
maxbands = max(strip.bands for strip in strips)
|
|
574
|
+
if minbands != maxbands:
|
|
575
|
+
strips = [strip[:minbands] for strip in strips]
|
|
576
|
+
# Persist the strips to temp files to build them into single objects;
|
|
577
|
+
# otherwise vips will use arbitrarily large amounts of memory
|
|
578
|
+
for sidx in range(len(strips)):
|
|
579
|
+
_pool_log(len(strips) - sidx, len(strips), 'resolving strips')
|
|
580
|
+
strip = strips[sidx]
|
|
581
|
+
vimgTemp = pyvips.Image.new_temp_file('%s.v')
|
|
582
|
+
strip.write(vimgTemp)
|
|
583
|
+
strips[sidx] = vimgTemp
|
|
528
584
|
img = strips[0]
|
|
529
585
|
for stripidx in range(1, len(strips)):
|
|
530
586
|
img = img.insert(strips[stripidx], 0, stripidx * _iterTileSize, expand=True)
|
|
531
587
|
_convert_via_vips(
|
|
532
|
-
img, frameOutputPath, tempPath, status='%d/%d' % (frame + 1, numFrames),
|
|
588
|
+
img, frameOutputPath, tempPath, status='%d/%d' % (frame + 1, numFrames),
|
|
589
|
+
preferredVipsCast=preferredVipsCast, **kwargs)
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def _output_type(lidata, keepFloat=False): # noqa
|
|
593
|
+
"""
|
|
594
|
+
Determine how to cast and scale vips data based on actual image contents.
|
|
595
|
+
"""
|
|
596
|
+
try:
|
|
597
|
+
intype = np.dtype(lidata['tilesource'].dtype)
|
|
598
|
+
except Exception:
|
|
599
|
+
return None
|
|
600
|
+
if intype == np.uint8 or intype == np.uint16:
|
|
601
|
+
return None
|
|
602
|
+
if keepFloat and np.issubdtype(intype, np.floating):
|
|
603
|
+
if intype == np.float16 or intype == np.float32:
|
|
604
|
+
return (pyvips.BandFormat.FLOAT, 0, 1)
|
|
605
|
+
return (pyvips.BandFormat.DOUBLE, 0, 1)
|
|
606
|
+
logger.debug('Checking data range')
|
|
607
|
+
minval = maxval = None
|
|
608
|
+
for frame in range(len(lidata['metadata'].get('frames', [0]))):
|
|
609
|
+
h = lidata['tilesource'].histogram(
|
|
610
|
+
onlyMinMax=True, output=dict(maxWidth=2048, maxHeight=2048),
|
|
611
|
+
resample=0, frame=frame)
|
|
612
|
+
if 'max' not in h:
|
|
613
|
+
continue
|
|
614
|
+
if maxval is None:
|
|
615
|
+
maxval = max(h['max'].tolist())
|
|
616
|
+
minval = min(h['min'].tolist())
|
|
617
|
+
else:
|
|
618
|
+
maxval = max(maxval, max(h['max'].tolist()))
|
|
619
|
+
minval = min(minval, min(h['min'].tolist()))
|
|
620
|
+
lidata['range'] = (minval, maxval)
|
|
621
|
+
logger.debug('Data range is [%r, %r]', minval, maxval)
|
|
622
|
+
if minval >= 0 and intype == np.int8:
|
|
623
|
+
return (pyvips.BandFormat.UCHAR, 0, 1)
|
|
624
|
+
if minval >= 0 and intype == np.int16:
|
|
625
|
+
return (pyvips.BandFormat.USHORT, 0, 1)
|
|
626
|
+
if minval >= 0 and maxval == 0:
|
|
627
|
+
return (pyvips.BandFormat.UCHAR, 0, 1)
|
|
628
|
+
if minval >= 0 and maxval <= 2 ** -8:
|
|
629
|
+
return (pyvips.BandFormat.USHORT, 0,
|
|
630
|
+
2 ** -(math.ceil(math.log2(maxval)) - 16) - 2 ** -math.ceil(math.log2(maxval)))
|
|
631
|
+
if minval >= 0 and maxval <= 1:
|
|
632
|
+
return (pyvips.BandFormat.USHORT, 0, 65535)
|
|
633
|
+
if minval >= 0 and maxval < 256:
|
|
634
|
+
return (pyvips.BandFormat.UCHAR, 0, 1)
|
|
635
|
+
if minval >= 0 and maxval < 65536:
|
|
636
|
+
return (pyvips.BandFormat.USHORT, 0, 1)
|
|
637
|
+
if minval >= 0:
|
|
638
|
+
return (pyvips.BandFormat.USHORT, 0,
|
|
639
|
+
2 ** -(math.ceil(math.log2(maxval)) - 16) - 2 ** -math.ceil(math.log2(maxval)))
|
|
640
|
+
if minval >= -2 ** -8 and maxval <= 2 ** -8:
|
|
641
|
+
return (pyvips.BandFormat.USHORT, 1,
|
|
642
|
+
2 ** -(math.ceil(math.log2(maxval)) - 15) - 2 ** -math.ceil(math.log2(maxval)))
|
|
643
|
+
if minval >= -1 and maxval <= 1:
|
|
644
|
+
return (pyvips.BandFormat.USHORT, 1, 32767)
|
|
645
|
+
if minval >= -32768 and maxval < 32768:
|
|
646
|
+
return (pyvips.BandFormat.USHORT, 32768, 1)
|
|
647
|
+
return (pyvips.BandFormat.USHORT, 0,
|
|
648
|
+
2 ** -(math.ceil(math.log2(max(-minval, maxval))) - 16) -
|
|
649
|
+
2 ** -math.ceil(math.log2(max(-minval, maxval))))
|
|
533
650
|
|
|
534
651
|
|
|
535
652
|
def _convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs):
|
|
@@ -544,23 +661,25 @@ def _convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs):
|
|
|
544
661
|
images.
|
|
545
662
|
"""
|
|
546
663
|
ts = lidata['tilesource']
|
|
664
|
+
lidata['_vips_cast'] = _output_type(lidata, kwargs.get('keepFloat', False))
|
|
547
665
|
numFrames = len(lidata['metadata'].get('frames', [0]))
|
|
548
666
|
outputList = []
|
|
549
667
|
tasks = []
|
|
550
|
-
pool = _get_thread_pool(memoryLimit=FrameMemoryEstimate, **kwargs)
|
|
551
668
|
startFrame = 0
|
|
552
669
|
endFrame = numFrames
|
|
553
670
|
if kwargs.get('onlyFrame') is not None and str(kwargs.get('onlyFrame')):
|
|
554
671
|
startFrame = int(kwargs.get('onlyFrame'))
|
|
555
672
|
endFrame = startFrame + 1
|
|
673
|
+
pool = _get_thread_pool(memoryLimit=FrameMemoryEstimate,
|
|
674
|
+
numItems=endFrame - startFrame, **kwargs)
|
|
556
675
|
for frame in range(startFrame, endFrame):
|
|
557
676
|
frameOutputPath = tempPath + '-%d-%s.tiff' % (
|
|
558
677
|
frame + 1, time.strftime('%Y%m%d-%H%M%S'))
|
|
559
678
|
_pool_add(tasks, (pool.submit(
|
|
560
679
|
_convert_large_image_frame, frame, numFrames, ts, frameOutputPath,
|
|
561
|
-
tempPath, **kwargs), ))
|
|
680
|
+
tempPath, lidata['_vips_cast'], pool._max_workers, **kwargs), ))
|
|
562
681
|
outputList.append(frameOutputPath)
|
|
563
|
-
_drain_pool(pool, tasks)
|
|
682
|
+
_drain_pool(pool, tasks, 'frames')
|
|
564
683
|
_output_tiff(outputList, outputPath, tempPath, lidata, **kwargs)
|
|
565
684
|
|
|
566
685
|
|
|
@@ -728,27 +847,27 @@ def _is_lossy(path, tiffinfo=None):
|
|
|
728
847
|
|
|
729
848
|
def _is_multiframe(path):
|
|
730
849
|
"""
|
|
731
|
-
Check if a path is a multiframe file.
|
|
850
|
+
Check if a path is a multiframe file according to vips.
|
|
732
851
|
|
|
733
852
|
:param path: The path to the file
|
|
734
853
|
:returns: True if multiframe.
|
|
735
854
|
"""
|
|
736
855
|
_import_pyvips()
|
|
737
856
|
try:
|
|
738
|
-
|
|
857
|
+
with _newFromFileLock:
|
|
858
|
+
image = pyvips.Image.new_from_file(path)
|
|
739
859
|
except Exception:
|
|
740
|
-
|
|
741
|
-
open(path, 'rb').read(1)
|
|
742
|
-
raise
|
|
743
|
-
except Exception:
|
|
744
|
-
logger.warning('Is the file reachable and readable? (%r)', path)
|
|
745
|
-
raise OSError(path) from None
|
|
860
|
+
return None
|
|
746
861
|
pages = 1
|
|
747
862
|
if 'n-pages' in image.get_fields():
|
|
748
863
|
pages = image.get_value('n-pages')
|
|
749
864
|
return pages > 1
|
|
750
865
|
|
|
751
866
|
|
|
867
|
+
def _is_new(path):
|
|
868
|
+
return os.path.basename(path).startswith(large_image.constants.NEW_IMAGE_PATH_FLAG)
|
|
869
|
+
|
|
870
|
+
|
|
752
871
|
def _list_possible_sizes(width, height):
|
|
753
872
|
"""
|
|
754
873
|
Given a width and height, return a list of possible sizes that could be
|
|
@@ -871,6 +990,8 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
|
|
|
871
990
|
primary ifds.
|
|
872
991
|
:param overwrite: if not True, throw an exception if the output path
|
|
873
992
|
already exists.
|
|
993
|
+
:param keepFloat: if True, keep float or double data types as they are, if
|
|
994
|
+
possible.
|
|
874
995
|
|
|
875
996
|
Additional optional parameters:
|
|
876
997
|
|
|
@@ -881,6 +1002,7 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
|
|
|
881
1002
|
|
|
882
1003
|
:returns: outputPath if successful
|
|
883
1004
|
"""
|
|
1005
|
+
logger._pool_log_starttime = time.time()
|
|
884
1006
|
if kwargs.get('_concurrency'):
|
|
885
1007
|
os.environ['VIPS_CONCURRENCY'] = str(_concurrency_to_value(**kwargs))
|
|
886
1008
|
geospatial = kwargs.get('geospatial')
|
|
@@ -889,7 +1011,7 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
|
|
|
889
1011
|
logger.debug('Is file geospatial: %r', geospatial)
|
|
890
1012
|
suffix = format_hook('adjust_params', geospatial, kwargs, **kwargs)
|
|
891
1013
|
if suffix is False:
|
|
892
|
-
return
|
|
1014
|
+
return None
|
|
893
1015
|
suffix = suffix or ('.tiff' if not geospatial else '.geo.tiff')
|
|
894
1016
|
if not outputPath:
|
|
895
1017
|
outputPath = os.path.splitext(inputPath)[0] + suffix
|
|
@@ -906,7 +1028,7 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
|
|
|
906
1028
|
except Exception:
|
|
907
1029
|
tiffinfo = None
|
|
908
1030
|
eightbit = _is_eightbit(inputPath, tiffinfo)
|
|
909
|
-
if not kwargs.get('compression'
|
|
1031
|
+
if not kwargs.get('compression'):
|
|
910
1032
|
kwargs = kwargs.copy()
|
|
911
1033
|
lossy = _is_lossy(inputPath, tiffinfo)
|
|
912
1034
|
logger.debug('Is file lossy: %r', lossy)
|
|
@@ -918,12 +1040,18 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
|
|
|
918
1040
|
else:
|
|
919
1041
|
with TemporaryDirectory() as tempDir:
|
|
920
1042
|
tempPath = os.path.join(tempDir, os.path.basename(outputPath))
|
|
921
|
-
lidata = _data_from_large_image(inputPath, tempPath, **kwargs)
|
|
1043
|
+
lidata = _data_from_large_image(str(inputPath), tempPath, **kwargs)
|
|
922
1044
|
logger.log(logging.DEBUG - 1, 'large_image information for %s: %r',
|
|
923
1045
|
inputPath, lidata)
|
|
924
|
-
if lidata and (
|
|
925
|
-
|
|
926
|
-
|
|
1046
|
+
if lidata and (_is_new(inputPath) or _is_multiframe(inputPath)):
|
|
1047
|
+
_convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs)
|
|
1048
|
+
elif lidata and (
|
|
1049
|
+
(len(lidata['metadata'].get('frames', [])) >= 2 and
|
|
1050
|
+
not _is_multiframe(inputPath)) or
|
|
1051
|
+
(np.dtype(lidata['tilesource'].dtype) != np.uint8 and
|
|
1052
|
+
np.dtype(lidata['tilesource'].dtype) != np.uint16) or
|
|
1053
|
+
not is_vips(inputPath, (lidata['metadata']['sizeX'], lidata['metadata']['sizeY']))
|
|
1054
|
+
):
|
|
927
1055
|
_convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs)
|
|
928
1056
|
elif _is_multiframe(inputPath):
|
|
929
1057
|
_generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
|
|
@@ -933,6 +1061,8 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
|
|
|
933
1061
|
except Exception:
|
|
934
1062
|
if lidata:
|
|
935
1063
|
_convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs)
|
|
1064
|
+
else:
|
|
1065
|
+
raise
|
|
936
1066
|
return outputPath
|
|
937
1067
|
|
|
938
1068
|
|
|
@@ -961,19 +1091,24 @@ def is_geospatial(path):
|
|
|
961
1091
|
return False
|
|
962
1092
|
|
|
963
1093
|
|
|
964
|
-
def is_vips(path):
|
|
1094
|
+
def is_vips(path, matchSize=None):
|
|
965
1095
|
"""
|
|
966
1096
|
Check if a path is readable by vips.
|
|
967
1097
|
|
|
968
1098
|
:param path: The path to the file
|
|
1099
|
+
:param matchSize: if not None, the image read by vips must be the specified
|
|
1100
|
+
(width, height) tuple in pixels.
|
|
969
1101
|
:returns: True if readable by vips.
|
|
970
1102
|
"""
|
|
971
1103
|
_import_pyvips()
|
|
972
1104
|
try:
|
|
973
|
-
|
|
1105
|
+
with _newFromFileLock:
|
|
1106
|
+
image = pyvips.Image.new_from_file(path)
|
|
974
1107
|
# image(0, 0) will throw if vips can't decode the image
|
|
975
1108
|
if not image.width or not image.height or image(0, 0) is None:
|
|
976
1109
|
return False
|
|
1110
|
+
if matchSize and (matchSize[0] != image.width or matchSize[1] != image.height):
|
|
1111
|
+
return False
|
|
977
1112
|
except Exception:
|
|
978
1113
|
return False
|
|
979
1114
|
return True
|
|
@@ -230,7 +230,7 @@ def main(args=sys.argv[1:]):
|
|
|
230
230
|
try:
|
|
231
231
|
import large_image
|
|
232
232
|
|
|
233
|
-
li_logger = large_image.config.
|
|
233
|
+
li_logger = large_image.config.getLogger()
|
|
234
234
|
li_logger.setLevel(max(1, logging.CRITICAL - (opts.verbose - opts.silent) * 10))
|
|
235
235
|
except ImportError:
|
|
236
236
|
pass
|
|
@@ -260,7 +260,7 @@ def main(args=sys.argv[1:]):
|
|
|
260
260
|
desc = json.loads(info['ifds'][0]['tags'][tifftools.Tag.ImageDescription.value]['data'])
|
|
261
261
|
except Exception:
|
|
262
262
|
logger.debug('Cannot generate statistics.')
|
|
263
|
-
return
|
|
263
|
+
return None
|
|
264
264
|
desc['large_image_converter']['conversion_stats'] = {
|
|
265
265
|
'time': end_time - start_time,
|
|
266
266
|
'filesize': os.path.getsize(dest),
|
|
@@ -1,25 +1,53 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: large-image-converter
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.33.4.dev39
|
|
4
4
|
Summary: Converter for Large Image.
|
|
5
5
|
Author: Kitware Inc
|
|
6
6
|
Author-email: kitware@kitware.com
|
|
7
|
-
License: Apache
|
|
7
|
+
License: Apache-2.0
|
|
8
8
|
Classifier: Development Status :: 5 - Production/Stable
|
|
9
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
9
|
Classifier: Topic :: Scientific/Engineering
|
|
11
10
|
Classifier: Intended Audience :: Science/Research
|
|
12
11
|
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
14
12
|
Classifier: Programming Language :: Python :: 3.9
|
|
15
13
|
Classifier: Programming Language :: Python :: 3.10
|
|
16
14
|
Classifier: Programming Language :: Python :: 3.11
|
|
17
15
|
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/x-rst
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: large-image-source-tiff>=1.33.4.dev39
|
|
21
|
+
Requires-Dist: numpy
|
|
22
|
+
Requires-Dist: psutil
|
|
23
|
+
Requires-Dist: pyvips
|
|
24
|
+
Requires-Dist: tifftools
|
|
19
25
|
Provides-Extra: jp2k
|
|
26
|
+
Requires-Dist: glymur; extra == "jp2k"
|
|
27
|
+
Provides-Extra: geospatial
|
|
28
|
+
Requires-Dist: gdal; extra == "geospatial"
|
|
20
29
|
Provides-Extra: sources
|
|
30
|
+
Requires-Dist: large-image[sources]>=1.33.4.dev39; extra == "sources"
|
|
21
31
|
Provides-Extra: stats
|
|
22
|
-
|
|
32
|
+
Requires-Dist: packaging; extra == "stats"
|
|
33
|
+
Requires-Dist: scikit-image; extra == "stats"
|
|
34
|
+
Provides-Extra: all
|
|
35
|
+
Requires-Dist: glymur; extra == "all"
|
|
36
|
+
Requires-Dist: gdal; extra == "all"
|
|
37
|
+
Requires-Dist: large-image[sources]>=1.33.4.dev39; extra == "all"
|
|
38
|
+
Requires-Dist: packaging; extra == "all"
|
|
39
|
+
Requires-Dist: scikit-image; extra == "all"
|
|
40
|
+
Dynamic: author
|
|
41
|
+
Dynamic: author-email
|
|
42
|
+
Dynamic: classifier
|
|
43
|
+
Dynamic: description
|
|
44
|
+
Dynamic: description-content-type
|
|
45
|
+
Dynamic: license
|
|
46
|
+
Dynamic: license-file
|
|
47
|
+
Dynamic: provides-extra
|
|
48
|
+
Dynamic: requires-dist
|
|
49
|
+
Dynamic: requires-python
|
|
50
|
+
Dynamic: summary
|
|
23
51
|
|
|
24
52
|
*********************
|
|
25
53
|
Large Image Converter
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
large-image-source-tiff>=1.33.4.dev39
|
|
2
|
+
numpy
|
|
3
|
+
psutil
|
|
4
|
+
pyvips
|
|
5
|
+
tifftools
|
|
6
|
+
|
|
7
|
+
[all]
|
|
8
|
+
glymur
|
|
9
|
+
gdal
|
|
10
|
+
large-image[sources]>=1.33.4.dev39
|
|
11
|
+
packaging
|
|
12
|
+
scikit-image
|
|
13
|
+
|
|
14
|
+
[geospatial]
|
|
15
|
+
gdal
|
|
16
|
+
|
|
17
|
+
[jp2k]
|
|
18
|
+
glymur
|
|
19
|
+
|
|
20
|
+
[sources]
|
|
21
|
+
large-image[sources]>=1.33.4.dev39
|
|
22
|
+
|
|
23
|
+
[stats]
|
|
24
|
+
packaging
|
|
25
|
+
scikit-image
|
|
@@ -9,54 +9,35 @@ description = 'Converter for Large Image.'
|
|
|
9
9
|
long_description = readme
|
|
10
10
|
|
|
11
11
|
|
|
12
|
-
def prerelease_local_scheme(version):
|
|
13
|
-
"""
|
|
14
|
-
Return local scheme version unless building on master in CircleCI.
|
|
15
|
-
|
|
16
|
-
This function returns the local scheme version number
|
|
17
|
-
(e.g. 0.0.0.dev<N>+g<HASH>) unless building on CircleCI for a
|
|
18
|
-
pre-release in which case it ignores the hash and produces a
|
|
19
|
-
PEP440 compliant pre-release version number (e.g. 0.0.0.dev<N>).
|
|
20
|
-
"""
|
|
21
|
-
from setuptools_scm.version import get_local_node_and_date
|
|
22
|
-
|
|
23
|
-
if os.getenv('CIRCLE_BRANCH') in ('master', ):
|
|
24
|
-
return ''
|
|
25
|
-
else:
|
|
26
|
-
return get_local_node_and_date(version)
|
|
27
|
-
|
|
28
|
-
|
|
29
12
|
try:
|
|
30
13
|
from setuptools_scm import get_version
|
|
31
14
|
|
|
32
|
-
version = get_version(root='../..'
|
|
15
|
+
version = get_version(root='../..')
|
|
33
16
|
limit_version = f'>={version}' if '+' not in version and not os.getenv('TOX_ENV_NAME') else ''
|
|
34
17
|
except (ImportError, LookupError):
|
|
35
18
|
limit_version = ''
|
|
36
19
|
|
|
37
20
|
setup(
|
|
38
21
|
name='large-image-converter',
|
|
39
|
-
use_scm_version={'root': '../..', 'local_scheme': prerelease_local_scheme,
|
|
40
|
-
'fallback_version': '0.0.0'},
|
|
41
22
|
description=description,
|
|
42
23
|
long_description=long_description,
|
|
43
|
-
|
|
24
|
+
long_description_content_type='text/x-rst',
|
|
25
|
+
license='Apache-2.0',
|
|
44
26
|
author='Kitware Inc',
|
|
45
27
|
author_email='kitware@kitware.com',
|
|
46
28
|
classifiers=[
|
|
47
29
|
'Development Status :: 5 - Production/Stable',
|
|
48
|
-
'License :: OSI Approved :: Apache Software License',
|
|
49
30
|
'Topic :: Scientific/Engineering',
|
|
50
31
|
'Intended Audience :: Science/Research',
|
|
51
32
|
'Programming Language :: Python :: 3',
|
|
52
|
-
'Programming Language :: Python :: 3.8',
|
|
53
33
|
'Programming Language :: Python :: 3.9',
|
|
54
34
|
'Programming Language :: Python :: 3.10',
|
|
55
35
|
'Programming Language :: Python :: 3.11',
|
|
56
36
|
'Programming Language :: Python :: 3.12',
|
|
37
|
+
'Programming Language :: Python :: 3.13',
|
|
57
38
|
],
|
|
39
|
+
python_requires='>=3.9',
|
|
58
40
|
install_requires=[
|
|
59
|
-
'gdal',
|
|
60
41
|
f'large-image-source-tiff{limit_version}',
|
|
61
42
|
'numpy',
|
|
62
43
|
'psutil',
|
|
@@ -67,6 +48,9 @@ setup(
|
|
|
67
48
|
'jp2k': [
|
|
68
49
|
'glymur',
|
|
69
50
|
],
|
|
51
|
+
'geospatial': [
|
|
52
|
+
'gdal',
|
|
53
|
+
],
|
|
70
54
|
'sources': [
|
|
71
55
|
f'large-image[sources]{limit_version}',
|
|
72
56
|
],
|
|
@@ -74,10 +58,16 @@ setup(
|
|
|
74
58
|
'packaging',
|
|
75
59
|
'scikit-image',
|
|
76
60
|
],
|
|
61
|
+
'all': [
|
|
62
|
+
'glymur',
|
|
63
|
+
'gdal',
|
|
64
|
+
f'large-image[sources]{limit_version}',
|
|
65
|
+
'packaging',
|
|
66
|
+
'scikit-image',
|
|
67
|
+
],
|
|
77
68
|
},
|
|
78
69
|
packages=find_packages(),
|
|
79
70
|
entry_points={
|
|
80
71
|
'console_scripts': ['large_image_converter = large_image_converter.__main__:main'],
|
|
81
72
|
},
|
|
82
|
-
python_requires='>=3.8',
|
|
83
73
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|