large-image-converter 1.26.3.dev10__tar.gz → 1.33.4.dev39__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/PKG-INFO +35 -7
  2. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter/__init__.py +176 -41
  3. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter/__main__.py +2 -2
  4. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter.egg-info/PKG-INFO +35 -7
  5. large_image_converter-1.33.4.dev39/large_image_converter.egg-info/requires.txt +25 -0
  6. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/pyproject.toml +4 -0
  7. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/setup.py +15 -25
  8. large-image-converter-1.26.3.dev10/large_image_converter.egg-info/requires.txt +0 -16
  9. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/LICENSE +0 -0
  10. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/README.rst +0 -0
  11. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter/format_aperio.py +0 -0
  12. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter.egg-info/SOURCES.txt +0 -0
  13. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter.egg-info/dependency_links.txt +0 -0
  14. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter.egg-info/entry_points.txt +0 -0
  15. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/large_image_converter.egg-info/top_level.txt +0 -0
  16. {large-image-converter-1.26.3.dev10 → large_image_converter-1.33.4.dev39}/setup.cfg +0 -0
@@ -1,25 +1,53 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: large-image-converter
3
- Version: 1.26.3.dev10
3
+ Version: 1.33.4.dev39
4
4
  Summary: Converter for Large Image.
5
5
  Author: Kitware Inc
6
6
  Author-email: kitware@kitware.com
7
- License: Apache Software License 2.0
7
+ License: Apache-2.0
8
8
  Classifier: Development Status :: 5 - Production/Stable
9
- Classifier: License :: OSI Approved :: Apache Software License
10
9
  Classifier: Topic :: Scientific/Engineering
11
10
  Classifier: Intended Audience :: Science/Research
12
11
  Classifier: Programming Language :: Python :: 3
13
- Classifier: Programming Language :: Python :: 3.8
14
12
  Classifier: Programming Language :: Python :: 3.9
15
13
  Classifier: Programming Language :: Python :: 3.10
16
14
  Classifier: Programming Language :: Python :: 3.11
17
15
  Classifier: Programming Language :: Python :: 3.12
18
- Requires-Python: >=3.8
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Requires-Python: >=3.9
18
+ Description-Content-Type: text/x-rst
19
+ License-File: LICENSE
20
+ Requires-Dist: large-image-source-tiff>=1.33.4.dev39
21
+ Requires-Dist: numpy
22
+ Requires-Dist: psutil
23
+ Requires-Dist: pyvips
24
+ Requires-Dist: tifftools
19
25
  Provides-Extra: jp2k
26
+ Requires-Dist: glymur; extra == "jp2k"
27
+ Provides-Extra: geospatial
28
+ Requires-Dist: gdal; extra == "geospatial"
20
29
  Provides-Extra: sources
30
+ Requires-Dist: large-image[sources]>=1.33.4.dev39; extra == "sources"
21
31
  Provides-Extra: stats
22
- License-File: LICENSE
32
+ Requires-Dist: packaging; extra == "stats"
33
+ Requires-Dist: scikit-image; extra == "stats"
34
+ Provides-Extra: all
35
+ Requires-Dist: glymur; extra == "all"
36
+ Requires-Dist: gdal; extra == "all"
37
+ Requires-Dist: large-image[sources]>=1.33.4.dev39; extra == "all"
38
+ Requires-Dist: packaging; extra == "all"
39
+ Requires-Dist: scikit-image; extra == "all"
40
+ Dynamic: author
41
+ Dynamic: author-email
42
+ Dynamic: classifier
43
+ Dynamic: description
44
+ Dynamic: description-content-type
45
+ Dynamic: license
46
+ Dynamic: license-file
47
+ Dynamic: provides-extra
48
+ Dynamic: requires-dist
49
+ Dynamic: requires-python
50
+ Dynamic: summary
23
51
 
24
52
  *********************
25
53
  Large Image Converter
@@ -14,10 +14,12 @@ from importlib.metadata import version as _importlib_version
14
14
  from tempfile import TemporaryDirectory
15
15
 
16
16
  import numpy as np
17
- import psutil
18
17
  import tifftools
19
18
 
20
19
  import large_image
20
+ from large_image.tilesource.utilities import (_gdalParameters,
21
+ _newFromFileLock, _vipsCast,
22
+ _vipsParameters)
21
23
 
22
24
  from . import format_aperio
23
25
 
@@ -76,9 +78,9 @@ def _data_from_large_image(path, outputPath, **kwargs):
76
78
  _import_pyvips()
77
79
  if not path.startswith('large_image://test'):
78
80
  try:
79
- ts = large_image.open(path)
81
+ ts = large_image.open(path, noCache=True)
80
82
  except Exception:
81
- return
83
+ return None
82
84
  else:
83
85
  import urllib.parse
84
86
 
@@ -107,7 +109,7 @@ def _data_from_large_image(path, outputPath, **kwargs):
107
109
  _pool_add(tasks, (pool.submit(
108
110
  _convert_via_vips, img, savePath, outputPath, mime=mime, forTiled=False), ))
109
111
  results['images'][key] = savePath
110
- _drain_pool(pool, tasks)
112
+ _drain_pool(pool, tasks, 'associated images')
111
113
  return results
112
114
 
113
115
 
@@ -129,7 +131,7 @@ def _generate_geotiff(inputPath, outputPath, **kwargs):
129
131
  """
130
132
  from osgeo import gdal, gdalconst
131
133
 
132
- cmdopt = large_image.tilesource.base._gdalParameters(**kwargs)
134
+ cmdopt = _gdalParameters(**kwargs)
133
135
  cmd = ['gdal_translate', inputPath, outputPath] + cmdopt
134
136
  logger.info('Convert to geotiff: %r', cmd)
135
137
  try:
@@ -163,7 +165,8 @@ def _generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
163
165
  """
164
166
  _import_pyvips()
165
167
 
166
- image = pyvips.Image.new_from_file(inputPath)
168
+ with _newFromFileLock:
169
+ image = pyvips.Image.new_from_file(inputPath)
167
170
  width = image.width
168
171
  height = image.height
169
172
  pages = 1
@@ -179,7 +182,8 @@ def _generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
179
182
  # Process each image separately to pyramidize it
180
183
  for page in range(pages):
181
184
  subInputPath = inputPath + '[page=%d]' % page
182
- subImage = pyvips.Image.new_from_file(subInputPath)
185
+ with _newFromFileLock:
186
+ subImage = pyvips.Image.new_from_file(subInputPath)
183
187
  imageSizes.append((subImage.width, subImage.height, subInputPath, page))
184
188
  if subImage.width != width or subImage.height != height:
185
189
  if subImage.width * subImage.height <= width * height:
@@ -214,7 +218,7 @@ def _generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
214
218
  _pool_add(tasks, (pool.submit(
215
219
  _convert_via_vips, subInputPath, savePath, tempPath, False), ))
216
220
  extraImages[key] = savePath
217
- _drain_pool(pool, tasks)
221
+ _drain_pool(pool, tasks, 'subpage')
218
222
  _output_tiff(outputList, outputPath, tempPath, lidata, extraImages, **kwargs)
219
223
 
220
224
 
@@ -245,7 +249,7 @@ def _generate_tiff(inputPath, outputPath, tempPath, lidata, **kwargs):
245
249
 
246
250
 
247
251
  def _convert_via_vips(inputPathOrBuffer, outputPath, tempPath, forTiled=True,
248
- status=None, **kwargs):
252
+ status=None, preferredVipsCast=None, **kwargs):
249
253
  """
250
254
  Convert a file, buffer, or vips image to a tiff file. This is equivalent
251
255
  to a vips command line of
@@ -259,11 +263,12 @@ def _convert_via_vips(inputPathOrBuffer, outputPath, tempPath, forTiled=True,
259
263
  also stores files in TMPDIR
260
264
  :param forTiled: True if the output should be tiled, false if not.
261
265
  :param status: an optional additional string to add to log messages.
266
+ :param preferredVipsCast: vips scaling parameters to use in a cast.
262
267
  :param kwargs: addition arguments that get passed to _vipsParameters
263
268
  and _convert_to_jp2k.
264
269
  """
265
270
  _import_pyvips()
266
- convertParams = large_image.tilesource.base._vipsParameters(forTiled, **kwargs)
271
+ convertParams = _vipsParameters(forTiled, **kwargs)
267
272
  status = (', ' + status) if status else ''
268
273
  if isinstance(inputPathOrBuffer, pyvips.vimage.Image):
269
274
  source = 'vips image'
@@ -273,23 +278,25 @@ def _convert_via_vips(inputPathOrBuffer, outputPath, tempPath, forTiled=True,
273
278
  image = pyvips.Image.new_from_buffer(inputPathOrBuffer, '')
274
279
  else:
275
280
  source = inputPathOrBuffer
276
- image = pyvips.Image.new_from_file(inputPathOrBuffer)
281
+ with _newFromFileLock:
282
+ image = pyvips.Image.new_from_file(inputPathOrBuffer)
277
283
  logger.info('Input: %s, Output: %s, Options: %r%s',
278
284
  source, outputPath, convertParams, status)
279
285
  image = image.autorot()
280
286
  adjusted = format_hook('modify_vips_image_before_output', image, convertParams, **kwargs)
281
287
  if adjusted is False:
282
288
  return
283
- elif adjusted:
289
+ if adjusted:
284
290
  image = adjusted
285
291
  if (convertParams['compression'] not in {'jpeg'} or
286
292
  image.interpretation != pyvips.Interpretation.SCRGB):
287
293
  # jp2k compression supports more than 8-bits per sample, but the
288
294
  # decompressor claims this is unsupported.
289
- image = large_image.tilesource.base._vipsCast(
295
+ image = _vipsCast(
290
296
  image,
291
297
  convertParams['compression'] in {'webp', 'jpeg'} or
292
- kwargs.get('compression') in {'jp2k'})
298
+ kwargs.get('compression') in {'jp2k'},
299
+ preferredVipsCast)
293
300
  # TODO: revisit the TMPDIR override; this is not thread safe
294
301
  # oldtmpdir = os.environ.get('TMPDIR')
295
302
  # os.environ['TMPDIR'] = os.path.dirname(tempPath)
@@ -341,10 +348,10 @@ def _concurrency_to_value(_concurrency=None, **kwargs):
341
348
  _concurrency = int(_concurrency) if str(_concurrency).isdigit() else 0
342
349
  if _concurrency > 0:
343
350
  return _concurrency
344
- return max(1, psutil.cpu_count(logical=True) + _concurrency)
351
+ return max(1, large_image.config.cpu_count(logical=True) + _concurrency)
345
352
 
346
353
 
347
- def _get_thread_pool(memoryLimit=None, **kwargs):
354
+ def _get_thread_pool(memoryLimit=None, parentConcurrency=None, numItems=None, **kwargs):
348
355
  """
349
356
  Allocate a thread pool based on the specific concurrency.
350
357
 
@@ -352,12 +359,37 @@ def _get_thread_pool(memoryLimit=None, **kwargs):
352
359
  process per memoryLimit bytes of total memory.
353
360
  """
354
361
  concurrency = _concurrency_to_value(**kwargs)
362
+ if parentConcurrency and parentConcurrency > 1 and concurrency > 1:
363
+ concurrency = max(1, int(math.ceil(concurrency / parentConcurrency)))
355
364
  if memoryLimit:
356
- concurrency = min(concurrency, psutil.virtual_memory().total // memoryLimit)
365
+ if parentConcurrency:
366
+ memoryLimit *= parentConcurrency
367
+ concurrency = min(concurrency, large_image.config.total_memory() // memoryLimit)
368
+ if numItems and numItems >= 1 and concurrency > numItems:
369
+ concurrency = numItems
357
370
  concurrency = max(1, concurrency)
358
371
  return concurrent.futures.ThreadPoolExecutor(max_workers=concurrency)
359
372
 
360
373
 
374
+ def _pool_log(left, total, label):
375
+ """
376
+ Log processing within a pool.
377
+
378
+ :param left: units left to process.
379
+ :param total: total units left to process.
380
+ :param label: label to log describing what is being processed.
381
+ """
382
+ if not hasattr(logger, '_pool_log_starttime'):
383
+ logger._pool_log_starttime = time.time()
384
+ if not hasattr(logger, '_pool_log_lastlog'):
385
+ logger._pool_log_lastlog = time.time()
386
+ if time.time() - logger._pool_log_lastlog < 10:
387
+ return
388
+ elapsed = time.time() - logger._pool_log_starttime
389
+ logger.debug('%d/%d %s left %4.2fs', left, total, label, elapsed)
390
+ logger._pool_log_lastlog = time.time()
391
+
392
+
361
393
  def _pool_add(tasks, newtask):
362
394
  """
363
395
  Add a new task to a pool, then drain any finished tasks at the start of the
@@ -376,7 +408,7 @@ def _pool_add(tasks, newtask):
376
408
  tasks.pop(0)
377
409
 
378
410
 
379
- def _drain_pool(pool, tasks):
411
+ def _drain_pool(pool, tasks, label=''):
380
412
  """
381
413
  Wait for all tasks in a pool to complete, then shutdown the pool.
382
414
 
@@ -384,6 +416,8 @@ def _drain_pool(pool, tasks):
384
416
  :param tasks: a list containing either lists or tuples, the last element
385
417
  of which is a task submitted to the pool. Altered.
386
418
  """
419
+ numtasks = len(tasks)
420
+ _pool_log(len(tasks), numtasks, label)
387
421
  while len(tasks):
388
422
  # This allows better stopping on a SIGTERM
389
423
  try:
@@ -391,6 +425,7 @@ def _drain_pool(pool, tasks):
391
425
  except concurrent.futures.TimeoutError:
392
426
  continue
393
427
  tasks.pop(0)
428
+ _pool_log(len(tasks), numtasks, label)
394
429
  pool.shutdown(False)
395
430
 
396
431
 
@@ -502,7 +537,9 @@ def _convert_large_image_tile(tilelock, strips, tile):
502
537
  strips[ty] = strips[ty].insert(vimg, x, 0, expand=True)
503
538
 
504
539
 
505
- def _convert_large_image_frame(frame, numFrames, ts, frameOutputPath, tempPath, **kwargs):
540
+ def _convert_large_image_frame(
541
+ frame, numFrames, ts, frameOutputPath, tempPath, preferredVipsCast=None,
542
+ parentConcurrency=None, **kwargs):
506
543
  """
507
544
  Convert a single frame from a large_image source. This parallelizes tile
508
545
  reads. Once all tiles are converted to a composited vips image, a tiff
@@ -513,23 +550,103 @@ def _convert_large_image_frame(frame, numFrames, ts, frameOutputPath, tempPath,
513
550
  :param ts: the open tile source.
514
551
  :param frameOutputPath: the destination name for the tiff file.
515
552
  :param tempPath: a temporary file in a temporary directory.
553
+ :param preferredVipsCast: vips scaling parameters to use in a cast.
554
+ :param parentConcurrency: amount of concurrency used by parent task.
516
555
  """
517
556
  # The iterator tile size is a balance between memory use and fewer calls
518
557
  # and file handles.
519
558
  _iterTileSize = 4096
520
559
  logger.info('Processing frame %d/%d', frame + 1, numFrames)
521
560
  strips = []
522
- pool = _get_thread_pool(**kwargs)
561
+ pool = _get_thread_pool(
562
+ memoryLimit=FrameMemoryEstimate,
563
+ # allow multiple tiles even if we are using all the cores, as it
564
+ # balances I/O and computation
565
+ parentConcurrency=(parentConcurrency // 2),
566
+ **kwargs)
523
567
  tasks = []
524
568
  tilelock = threading.Lock()
525
569
  for tile in ts.tileIterator(tile_size=dict(width=_iterTileSize), frame=frame):
526
570
  _pool_add(tasks, (pool.submit(_convert_large_image_tile, tilelock, strips, tile), ))
527
- _drain_pool(pool, tasks)
571
+ _drain_pool(pool, tasks, f'tiles from frame {frame + 1}/{numFrames}')
572
+ minbands = min(strip.bands for strip in strips)
573
+ maxbands = max(strip.bands for strip in strips)
574
+ if minbands != maxbands:
575
+ strips = [strip[:minbands] for strip in strips]
576
+ # Persist the strips to temp files to build them into single objects;
577
+ # otherwise vips will use arbitrarily large amounts of memory
578
+ for sidx in range(len(strips)):
579
+ _pool_log(len(strips) - sidx, len(strips), 'resolving strips')
580
+ strip = strips[sidx]
581
+ vimgTemp = pyvips.Image.new_temp_file('%s.v')
582
+ strip.write(vimgTemp)
583
+ strips[sidx] = vimgTemp
528
584
  img = strips[0]
529
585
  for stripidx in range(1, len(strips)):
530
586
  img = img.insert(strips[stripidx], 0, stripidx * _iterTileSize, expand=True)
531
587
  _convert_via_vips(
532
- img, frameOutputPath, tempPath, status='%d/%d' % (frame + 1, numFrames), **kwargs)
588
+ img, frameOutputPath, tempPath, status='%d/%d' % (frame + 1, numFrames),
589
+ preferredVipsCast=preferredVipsCast, **kwargs)
590
+
591
+
592
+ def _output_type(lidata, keepFloat=False): # noqa
593
+ """
594
+ Determine how to cast and scale vips data based on actual image contents.
595
+ """
596
+ try:
597
+ intype = np.dtype(lidata['tilesource'].dtype)
598
+ except Exception:
599
+ return None
600
+ if intype == np.uint8 or intype == np.uint16:
601
+ return None
602
+ if keepFloat and np.issubdtype(intype, np.floating):
603
+ if intype == np.float16 or intype == np.float32:
604
+ return (pyvips.BandFormat.FLOAT, 0, 1)
605
+ return (pyvips.BandFormat.DOUBLE, 0, 1)
606
+ logger.debug('Checking data range')
607
+ minval = maxval = None
608
+ for frame in range(len(lidata['metadata'].get('frames', [0]))):
609
+ h = lidata['tilesource'].histogram(
610
+ onlyMinMax=True, output=dict(maxWidth=2048, maxHeight=2048),
611
+ resample=0, frame=frame)
612
+ if 'max' not in h:
613
+ continue
614
+ if maxval is None:
615
+ maxval = max(h['max'].tolist())
616
+ minval = min(h['min'].tolist())
617
+ else:
618
+ maxval = max(maxval, max(h['max'].tolist()))
619
+ minval = min(minval, min(h['min'].tolist()))
620
+ lidata['range'] = (minval, maxval)
621
+ logger.debug('Data range is [%r, %r]', minval, maxval)
622
+ if minval >= 0 and intype == np.int8:
623
+ return (pyvips.BandFormat.UCHAR, 0, 1)
624
+ if minval >= 0 and intype == np.int16:
625
+ return (pyvips.BandFormat.USHORT, 0, 1)
626
+ if minval >= 0 and maxval == 0:
627
+ return (pyvips.BandFormat.UCHAR, 0, 1)
628
+ if minval >= 0 and maxval <= 2 ** -8:
629
+ return (pyvips.BandFormat.USHORT, 0,
630
+ 2 ** -(math.ceil(math.log2(maxval)) - 16) - 2 ** -math.ceil(math.log2(maxval)))
631
+ if minval >= 0 and maxval <= 1:
632
+ return (pyvips.BandFormat.USHORT, 0, 65535)
633
+ if minval >= 0 and maxval < 256:
634
+ return (pyvips.BandFormat.UCHAR, 0, 1)
635
+ if minval >= 0 and maxval < 65536:
636
+ return (pyvips.BandFormat.USHORT, 0, 1)
637
+ if minval >= 0:
638
+ return (pyvips.BandFormat.USHORT, 0,
639
+ 2 ** -(math.ceil(math.log2(maxval)) - 16) - 2 ** -math.ceil(math.log2(maxval)))
640
+ if minval >= -2 ** -8 and maxval <= 2 ** -8:
641
+ return (pyvips.BandFormat.USHORT, 1,
642
+ 2 ** -(math.ceil(math.log2(maxval)) - 15) - 2 ** -math.ceil(math.log2(maxval)))
643
+ if minval >= -1 and maxval <= 1:
644
+ return (pyvips.BandFormat.USHORT, 1, 32767)
645
+ if minval >= -32768 and maxval < 32768:
646
+ return (pyvips.BandFormat.USHORT, 32768, 1)
647
+ return (pyvips.BandFormat.USHORT, 0,
648
+ 2 ** -(math.ceil(math.log2(max(-minval, maxval))) - 16) -
649
+ 2 ** -math.ceil(math.log2(max(-minval, maxval))))
533
650
 
534
651
 
535
652
  def _convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs):
@@ -544,23 +661,25 @@ def _convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs):
544
661
  images.
545
662
  """
546
663
  ts = lidata['tilesource']
664
+ lidata['_vips_cast'] = _output_type(lidata, kwargs.get('keepFloat', False))
547
665
  numFrames = len(lidata['metadata'].get('frames', [0]))
548
666
  outputList = []
549
667
  tasks = []
550
- pool = _get_thread_pool(memoryLimit=FrameMemoryEstimate, **kwargs)
551
668
  startFrame = 0
552
669
  endFrame = numFrames
553
670
  if kwargs.get('onlyFrame') is not None and str(kwargs.get('onlyFrame')):
554
671
  startFrame = int(kwargs.get('onlyFrame'))
555
672
  endFrame = startFrame + 1
673
+ pool = _get_thread_pool(memoryLimit=FrameMemoryEstimate,
674
+ numItems=endFrame - startFrame, **kwargs)
556
675
  for frame in range(startFrame, endFrame):
557
676
  frameOutputPath = tempPath + '-%d-%s.tiff' % (
558
677
  frame + 1, time.strftime('%Y%m%d-%H%M%S'))
559
678
  _pool_add(tasks, (pool.submit(
560
679
  _convert_large_image_frame, frame, numFrames, ts, frameOutputPath,
561
- tempPath, **kwargs), ))
680
+ tempPath, lidata['_vips_cast'], pool._max_workers, **kwargs), ))
562
681
  outputList.append(frameOutputPath)
563
- _drain_pool(pool, tasks)
682
+ _drain_pool(pool, tasks, 'frames')
564
683
  _output_tiff(outputList, outputPath, tempPath, lidata, **kwargs)
565
684
 
566
685
 
@@ -728,27 +847,27 @@ def _is_lossy(path, tiffinfo=None):
728
847
 
729
848
  def _is_multiframe(path):
730
849
  """
731
- Check if a path is a multiframe file.
850
+ Check if a path is a multiframe file according to vips.
732
851
 
733
852
  :param path: The path to the file
734
853
  :returns: True if multiframe.
735
854
  """
736
855
  _import_pyvips()
737
856
  try:
738
- image = pyvips.Image.new_from_file(path)
857
+ with _newFromFileLock:
858
+ image = pyvips.Image.new_from_file(path)
739
859
  except Exception:
740
- try:
741
- open(path, 'rb').read(1)
742
- raise
743
- except Exception:
744
- logger.warning('Is the file reachable and readable? (%r)', path)
745
- raise OSError(path) from None
860
+ return None
746
861
  pages = 1
747
862
  if 'n-pages' in image.get_fields():
748
863
  pages = image.get_value('n-pages')
749
864
  return pages > 1
750
865
 
751
866
 
867
+ def _is_new(path):
868
+ return os.path.basename(path).startswith(large_image.constants.NEW_IMAGE_PATH_FLAG)
869
+
870
+
752
871
  def _list_possible_sizes(width, height):
753
872
  """
754
873
  Given a width and height, return a list of possible sizes that could be
@@ -871,6 +990,8 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
871
990
  primary ifds.
872
991
  :param overwrite: if not True, throw an exception if the output path
873
992
  already exists.
993
+ :param keepFloat: if True, keep float or double data types as they are, if
994
+ possible.
874
995
 
875
996
  Additional optional parameters:
876
997
 
@@ -881,6 +1002,7 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
881
1002
 
882
1003
  :returns: outputPath if successful
883
1004
  """
1005
+ logger._pool_log_starttime = time.time()
884
1006
  if kwargs.get('_concurrency'):
885
1007
  os.environ['VIPS_CONCURRENCY'] = str(_concurrency_to_value(**kwargs))
886
1008
  geospatial = kwargs.get('geospatial')
@@ -889,7 +1011,7 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
889
1011
  logger.debug('Is file geospatial: %r', geospatial)
890
1012
  suffix = format_hook('adjust_params', geospatial, kwargs, **kwargs)
891
1013
  if suffix is False:
892
- return
1014
+ return None
893
1015
  suffix = suffix or ('.tiff' if not geospatial else '.geo.tiff')
894
1016
  if not outputPath:
895
1017
  outputPath = os.path.splitext(inputPath)[0] + suffix
@@ -906,7 +1028,7 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
906
1028
  except Exception:
907
1029
  tiffinfo = None
908
1030
  eightbit = _is_eightbit(inputPath, tiffinfo)
909
- if not kwargs.get('compression', None):
1031
+ if not kwargs.get('compression'):
910
1032
  kwargs = kwargs.copy()
911
1033
  lossy = _is_lossy(inputPath, tiffinfo)
912
1034
  logger.debug('Is file lossy: %r', lossy)
@@ -918,12 +1040,18 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
918
1040
  else:
919
1041
  with TemporaryDirectory() as tempDir:
920
1042
  tempPath = os.path.join(tempDir, os.path.basename(outputPath))
921
- lidata = _data_from_large_image(inputPath, tempPath, **kwargs)
1043
+ lidata = _data_from_large_image(str(inputPath), tempPath, **kwargs)
922
1044
  logger.log(logging.DEBUG - 1, 'large_image information for %s: %r',
923
1045
  inputPath, lidata)
924
- if lidata and (not is_vips(inputPath) or (
925
- len(lidata['metadata'].get('frames', [])) >= 2 and
926
- not _is_multiframe(inputPath))):
1046
+ if lidata and (_is_new(inputPath) or _is_multiframe(inputPath)):
1047
+ _convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs)
1048
+ elif lidata and (
1049
+ (len(lidata['metadata'].get('frames', [])) >= 2 and
1050
+ not _is_multiframe(inputPath)) or
1051
+ (np.dtype(lidata['tilesource'].dtype) != np.uint8 and
1052
+ np.dtype(lidata['tilesource'].dtype) != np.uint16) or
1053
+ not is_vips(inputPath, (lidata['metadata']['sizeX'], lidata['metadata']['sizeY']))
1054
+ ):
927
1055
  _convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs)
928
1056
  elif _is_multiframe(inputPath):
929
1057
  _generate_multiframe_tiff(inputPath, outputPath, tempPath, lidata, **kwargs)
@@ -933,6 +1061,8 @@ def convert(inputPath, outputPath=None, **kwargs): # noqa: C901
933
1061
  except Exception:
934
1062
  if lidata:
935
1063
  _convert_large_image(inputPath, outputPath, tempPath, lidata, **kwargs)
1064
+ else:
1065
+ raise
936
1066
  return outputPath
937
1067
 
938
1068
 
@@ -961,19 +1091,24 @@ def is_geospatial(path):
961
1091
  return False
962
1092
 
963
1093
 
964
- def is_vips(path):
1094
+ def is_vips(path, matchSize=None):
965
1095
  """
966
1096
  Check if a path is readable by vips.
967
1097
 
968
1098
  :param path: The path to the file
1099
+ :param matchSize: if not None, the image read by vips must be the specified
1100
+ (width, height) tuple in pixels.
969
1101
  :returns: True if readable by vips.
970
1102
  """
971
1103
  _import_pyvips()
972
1104
  try:
973
- image = pyvips.Image.new_from_file(path)
1105
+ with _newFromFileLock:
1106
+ image = pyvips.Image.new_from_file(path)
974
1107
  # image(0, 0) will throw if vips can't decode the image
975
1108
  if not image.width or not image.height or image(0, 0) is None:
976
1109
  return False
1110
+ if matchSize and (matchSize[0] != image.width or matchSize[1] != image.height):
1111
+ return False
977
1112
  except Exception:
978
1113
  return False
979
1114
  return True
@@ -230,7 +230,7 @@ def main(args=sys.argv[1:]):
230
230
  try:
231
231
  import large_image
232
232
 
233
- li_logger = large_image.config.getConfig('logger')
233
+ li_logger = large_image.config.getLogger()
234
234
  li_logger.setLevel(max(1, logging.CRITICAL - (opts.verbose - opts.silent) * 10))
235
235
  except ImportError:
236
236
  pass
@@ -260,7 +260,7 @@ def main(args=sys.argv[1:]):
260
260
  desc = json.loads(info['ifds'][0]['tags'][tifftools.Tag.ImageDescription.value]['data'])
261
261
  except Exception:
262
262
  logger.debug('Cannot generate statistics.')
263
- return
263
+ return None
264
264
  desc['large_image_converter']['conversion_stats'] = {
265
265
  'time': end_time - start_time,
266
266
  'filesize': os.path.getsize(dest),
@@ -1,25 +1,53 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: large-image-converter
3
- Version: 1.26.3.dev10
3
+ Version: 1.33.4.dev39
4
4
  Summary: Converter for Large Image.
5
5
  Author: Kitware Inc
6
6
  Author-email: kitware@kitware.com
7
- License: Apache Software License 2.0
7
+ License: Apache-2.0
8
8
  Classifier: Development Status :: 5 - Production/Stable
9
- Classifier: License :: OSI Approved :: Apache Software License
10
9
  Classifier: Topic :: Scientific/Engineering
11
10
  Classifier: Intended Audience :: Science/Research
12
11
  Classifier: Programming Language :: Python :: 3
13
- Classifier: Programming Language :: Python :: 3.8
14
12
  Classifier: Programming Language :: Python :: 3.9
15
13
  Classifier: Programming Language :: Python :: 3.10
16
14
  Classifier: Programming Language :: Python :: 3.11
17
15
  Classifier: Programming Language :: Python :: 3.12
18
- Requires-Python: >=3.8
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Requires-Python: >=3.9
18
+ Description-Content-Type: text/x-rst
19
+ License-File: LICENSE
20
+ Requires-Dist: large-image-source-tiff>=1.33.4.dev39
21
+ Requires-Dist: numpy
22
+ Requires-Dist: psutil
23
+ Requires-Dist: pyvips
24
+ Requires-Dist: tifftools
19
25
  Provides-Extra: jp2k
26
+ Requires-Dist: glymur; extra == "jp2k"
27
+ Provides-Extra: geospatial
28
+ Requires-Dist: gdal; extra == "geospatial"
20
29
  Provides-Extra: sources
30
+ Requires-Dist: large-image[sources]>=1.33.4.dev39; extra == "sources"
21
31
  Provides-Extra: stats
22
- License-File: LICENSE
32
+ Requires-Dist: packaging; extra == "stats"
33
+ Requires-Dist: scikit-image; extra == "stats"
34
+ Provides-Extra: all
35
+ Requires-Dist: glymur; extra == "all"
36
+ Requires-Dist: gdal; extra == "all"
37
+ Requires-Dist: large-image[sources]>=1.33.4.dev39; extra == "all"
38
+ Requires-Dist: packaging; extra == "all"
39
+ Requires-Dist: scikit-image; extra == "all"
40
+ Dynamic: author
41
+ Dynamic: author-email
42
+ Dynamic: classifier
43
+ Dynamic: description
44
+ Dynamic: description-content-type
45
+ Dynamic: license
46
+ Dynamic: license-file
47
+ Dynamic: provides-extra
48
+ Dynamic: requires-dist
49
+ Dynamic: requires-python
50
+ Dynamic: summary
23
51
 
24
52
  *********************
25
53
  Large Image Converter
@@ -0,0 +1,25 @@
1
+ large-image-source-tiff>=1.33.4.dev39
2
+ numpy
3
+ psutil
4
+ pyvips
5
+ tifftools
6
+
7
+ [all]
8
+ glymur
9
+ gdal
10
+ large-image[sources]>=1.33.4.dev39
11
+ packaging
12
+ scikit-image
13
+
14
+ [geospatial]
15
+ gdal
16
+
17
+ [jp2k]
18
+ glymur
19
+
20
+ [sources]
21
+ large-image[sources]>=1.33.4.dev39
22
+
23
+ [stats]
24
+ packaging
25
+ scikit-image
@@ -1,3 +1,7 @@
1
1
  [build-system]
2
2
  requires = ["setuptools", "setuptools-scm"]
3
3
  build-backend = "setuptools.build_meta"
4
+
5
+ [tool.setuptools_scm]
6
+ fallback_version = "0.0.0"
7
+ root = "../.."
@@ -9,54 +9,35 @@ description = 'Converter for Large Image.'
9
9
  long_description = readme
10
10
 
11
11
 
12
- def prerelease_local_scheme(version):
13
- """
14
- Return local scheme version unless building on master in CircleCI.
15
-
16
- This function returns the local scheme version number
17
- (e.g. 0.0.0.dev<N>+g<HASH>) unless building on CircleCI for a
18
- pre-release in which case it ignores the hash and produces a
19
- PEP440 compliant pre-release version number (e.g. 0.0.0.dev<N>).
20
- """
21
- from setuptools_scm.version import get_local_node_and_date
22
-
23
- if os.getenv('CIRCLE_BRANCH') in ('master', ):
24
- return ''
25
- else:
26
- return get_local_node_and_date(version)
27
-
28
-
29
12
  try:
30
13
  from setuptools_scm import get_version
31
14
 
32
- version = get_version(root='../..', local_scheme=prerelease_local_scheme)
15
+ version = get_version(root='../..')
33
16
  limit_version = f'>={version}' if '+' not in version and not os.getenv('TOX_ENV_NAME') else ''
34
17
  except (ImportError, LookupError):
35
18
  limit_version = ''
36
19
 
37
20
  setup(
38
21
  name='large-image-converter',
39
- use_scm_version={'root': '../..', 'local_scheme': prerelease_local_scheme,
40
- 'fallback_version': '0.0.0'},
41
22
  description=description,
42
23
  long_description=long_description,
43
- license='Apache Software License 2.0',
24
+ long_description_content_type='text/x-rst',
25
+ license='Apache-2.0',
44
26
  author='Kitware Inc',
45
27
  author_email='kitware@kitware.com',
46
28
  classifiers=[
47
29
  'Development Status :: 5 - Production/Stable',
48
- 'License :: OSI Approved :: Apache Software License',
49
30
  'Topic :: Scientific/Engineering',
50
31
  'Intended Audience :: Science/Research',
51
32
  'Programming Language :: Python :: 3',
52
- 'Programming Language :: Python :: 3.8',
53
33
  'Programming Language :: Python :: 3.9',
54
34
  'Programming Language :: Python :: 3.10',
55
35
  'Programming Language :: Python :: 3.11',
56
36
  'Programming Language :: Python :: 3.12',
37
+ 'Programming Language :: Python :: 3.13',
57
38
  ],
39
+ python_requires='>=3.9',
58
40
  install_requires=[
59
- 'gdal',
60
41
  f'large-image-source-tiff{limit_version}',
61
42
  'numpy',
62
43
  'psutil',
@@ -67,6 +48,9 @@ setup(
67
48
  'jp2k': [
68
49
  'glymur',
69
50
  ],
51
+ 'geospatial': [
52
+ 'gdal',
53
+ ],
70
54
  'sources': [
71
55
  f'large-image[sources]{limit_version}',
72
56
  ],
@@ -74,10 +58,16 @@ setup(
74
58
  'packaging',
75
59
  'scikit-image',
76
60
  ],
61
+ 'all': [
62
+ 'glymur',
63
+ 'gdal',
64
+ f'large-image[sources]{limit_version}',
65
+ 'packaging',
66
+ 'scikit-image',
67
+ ],
77
68
  },
78
69
  packages=find_packages(),
79
70
  entry_points={
80
71
  'console_scripts': ['large_image_converter = large_image_converter.__main__:main'],
81
72
  },
82
- python_requires='>=3.8',
83
73
  )
@@ -1,16 +0,0 @@
1
- gdal
2
- large-image-source-tiff>=1.26.3.dev10
3
- numpy
4
- psutil
5
- pyvips
6
- tifftools
7
-
8
- [jp2k]
9
- glymur
10
-
11
- [sources]
12
- large-image[sources]>=1.26.3.dev10
13
-
14
- [stats]
15
- packaging
16
- scikit-image