nabu 2024.1.9__py3-none-any.whl → 2024.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. nabu/__init__.py +1 -1
  2. nabu/app/bootstrap.py +2 -3
  3. nabu/app/cast_volume.py +4 -2
  4. nabu/app/cli_configs.py +5 -0
  5. nabu/app/composite_cor.py +1 -1
  6. nabu/app/create_distortion_map_from_poly.py +5 -6
  7. nabu/app/diag_to_pix.py +7 -19
  8. nabu/app/diag_to_rot.py +14 -29
  9. nabu/app/double_flatfield.py +32 -44
  10. nabu/app/parse_reconstruction_log.py +3 -0
  11. nabu/app/reconstruct.py +53 -15
  12. nabu/app/reconstruct_helical.py +2 -2
  13. nabu/app/stitching.py +27 -13
  14. nabu/app/tests/test_reduce_dark_flat.py +4 -1
  15. nabu/cuda/kernel.py +11 -2
  16. nabu/cuda/processing.py +2 -2
  17. nabu/cuda/src/cone.cu +77 -0
  18. nabu/cuda/src/hierarchical_backproj.cu +271 -0
  19. nabu/cuda/utils.py +0 -6
  20. nabu/estimation/alignment.py +5 -19
  21. nabu/estimation/cor.py +173 -599
  22. nabu/estimation/cor_sino.py +356 -26
  23. nabu/estimation/focus.py +63 -11
  24. nabu/estimation/tests/test_cor.py +124 -58
  25. nabu/estimation/tests/test_focus.py +6 -6
  26. nabu/estimation/tilt.py +2 -1
  27. nabu/estimation/utils.py +5 -33
  28. nabu/io/__init__.py +1 -1
  29. nabu/io/cast_volume.py +1 -1
  30. nabu/io/reader.py +416 -21
  31. nabu/io/tests/test_readers.py +422 -0
  32. nabu/io/tests/test_writers.py +1 -102
  33. nabu/io/writer.py +4 -433
  34. nabu/opencl/kernel.py +14 -3
  35. nabu/opencl/processing.py +8 -0
  36. nabu/pipeline/config_validators.py +5 -2
  37. nabu/pipeline/datadump.py +12 -5
  38. nabu/pipeline/estimators.py +162 -188
  39. nabu/pipeline/fullfield/chunked.py +168 -92
  40. nabu/pipeline/fullfield/chunked_cuda.py +7 -3
  41. nabu/pipeline/fullfield/computations.py +2 -7
  42. nabu/pipeline/fullfield/dataset_validator.py +0 -4
  43. nabu/pipeline/fullfield/nabu_config.py +37 -13
  44. nabu/pipeline/fullfield/processconfig.py +22 -13
  45. nabu/pipeline/fullfield/reconstruction.py +13 -9
  46. nabu/pipeline/helical/helical_chunked_regridded.py +1 -1
  47. nabu/pipeline/helical/helical_chunked_regridded_cuda.py +1 -0
  48. nabu/pipeline/helical/helical_reconstruction.py +1 -1
  49. nabu/pipeline/params.py +21 -1
  50. nabu/pipeline/processconfig.py +1 -12
  51. nabu/pipeline/reader.py +146 -0
  52. nabu/pipeline/tests/test_estimators.py +44 -72
  53. nabu/pipeline/utils.py +4 -2
  54. nabu/pipeline/writer.py +10 -2
  55. nabu/preproc/ccd_cuda.py +1 -1
  56. nabu/preproc/ctf.py +14 -7
  57. nabu/preproc/ctf_cuda.py +2 -3
  58. nabu/preproc/double_flatfield.py +5 -12
  59. nabu/preproc/double_flatfield_cuda.py +2 -2
  60. nabu/preproc/flatfield.py +5 -1
  61. nabu/preproc/flatfield_cuda.py +5 -1
  62. nabu/preproc/phase.py +24 -73
  63. nabu/preproc/phase_cuda.py +5 -8
  64. nabu/preproc/tests/test_ctf.py +11 -7
  65. nabu/preproc/tests/test_flatfield.py +67 -122
  66. nabu/preproc/tests/test_paganin.py +54 -30
  67. nabu/processing/azim.py +206 -0
  68. nabu/processing/convolution_cuda.py +1 -1
  69. nabu/processing/fft_cuda.py +15 -17
  70. nabu/processing/histogram.py +2 -0
  71. nabu/processing/histogram_cuda.py +2 -1
  72. nabu/processing/kernel_base.py +3 -0
  73. nabu/processing/muladd_cuda.py +1 -0
  74. nabu/processing/padding_opencl.py +1 -1
  75. nabu/processing/roll_opencl.py +1 -0
  76. nabu/processing/rotation_cuda.py +2 -2
  77. nabu/processing/tests/test_fft.py +17 -10
  78. nabu/processing/unsharp_cuda.py +1 -1
  79. nabu/reconstruction/cone.py +104 -40
  80. nabu/reconstruction/fbp.py +3 -0
  81. nabu/reconstruction/fbp_base.py +7 -2
  82. nabu/reconstruction/filtering.py +20 -7
  83. nabu/reconstruction/filtering_cuda.py +7 -1
  84. nabu/reconstruction/hbp.py +424 -0
  85. nabu/reconstruction/mlem.py +99 -0
  86. nabu/reconstruction/reconstructor.py +2 -0
  87. nabu/reconstruction/rings_cuda.py +19 -19
  88. nabu/reconstruction/sinogram_cuda.py +1 -0
  89. nabu/reconstruction/sinogram_opencl.py +3 -1
  90. nabu/reconstruction/tests/test_cone.py +10 -5
  91. nabu/reconstruction/tests/test_deringer.py +7 -6
  92. nabu/reconstruction/tests/test_fbp.py +124 -10
  93. nabu/reconstruction/tests/test_filtering.py +13 -11
  94. nabu/reconstruction/tests/test_halftomo.py +30 -4
  95. nabu/reconstruction/tests/test_mlem.py +91 -0
  96. nabu/reconstruction/tests/test_reconstructor.py +8 -3
  97. nabu/resources/dataset_analyzer.py +142 -92
  98. nabu/resources/gpu.py +1 -0
  99. nabu/resources/nxflatfield.py +134 -125
  100. nabu/resources/templates/id16a_fluo.conf +42 -0
  101. nabu/resources/tests/test_extract.py +10 -0
  102. nabu/resources/tests/test_nxflatfield.py +2 -2
  103. nabu/stitching/alignment.py +80 -24
  104. nabu/stitching/config.py +105 -68
  105. nabu/stitching/definitions.py +1 -0
  106. nabu/stitching/frame_composition.py +68 -60
  107. nabu/stitching/overlap.py +91 -51
  108. nabu/stitching/single_axis_stitching.py +32 -0
  109. nabu/stitching/slurm_utils.py +6 -6
  110. nabu/stitching/stitcher/__init__.py +0 -0
  111. nabu/stitching/stitcher/base.py +124 -0
  112. nabu/stitching/stitcher/dumper/__init__.py +3 -0
  113. nabu/stitching/stitcher/dumper/base.py +94 -0
  114. nabu/stitching/stitcher/dumper/postprocessing.py +356 -0
  115. nabu/stitching/stitcher/dumper/preprocessing.py +60 -0
  116. nabu/stitching/stitcher/post_processing.py +555 -0
  117. nabu/stitching/stitcher/pre_processing.py +1068 -0
  118. nabu/stitching/stitcher/single_axis.py +484 -0
  119. nabu/stitching/stitcher/stitcher.py +0 -0
  120. nabu/stitching/stitcher/y_stitcher.py +13 -0
  121. nabu/stitching/stitcher/z_stitcher.py +45 -0
  122. nabu/stitching/stitcher_2D.py +278 -0
  123. nabu/stitching/tests/test_config.py +12 -37
  124. nabu/stitching/tests/test_frame_composition.py +33 -59
  125. nabu/stitching/tests/test_overlap.py +149 -7
  126. nabu/stitching/tests/test_utils.py +1 -1
  127. nabu/stitching/tests/test_y_preprocessing_stitching.py +132 -0
  128. nabu/stitching/tests/{test_z_stitching.py → test_z_postprocessing_stitching.py} +167 -561
  129. nabu/stitching/tests/test_z_preprocessing_stitching.py +431 -0
  130. nabu/stitching/utils/__init__.py +1 -0
  131. nabu/stitching/utils/post_processing.py +281 -0
  132. nabu/stitching/utils/tests/test_post-processing.py +21 -0
  133. nabu/stitching/{utils.py → utils/utils.py} +79 -52
  134. nabu/stitching/y_stitching.py +27 -0
  135. nabu/stitching/z_stitching.py +32 -2263
  136. nabu/testutils.py +1 -152
  137. nabu/thirdparty/tomocupy_remove_stripe.py +43 -9
  138. nabu/utils.py +158 -61
  139. {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/METADATA +10 -3
  140. {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/RECORD +144 -121
  141. nabu/io/tiffwriter_zmm.py +0 -99
  142. nabu/pipeline/fallback_utils.py +0 -149
  143. nabu/pipeline/helical/tests/test_accumulator.py +0 -158
  144. nabu/pipeline/helical/tests/test_pipeline_elements_full.py +0 -355
  145. nabu/pipeline/helical/tests/test_strategy.py +0 -61
  146. nabu/pipeline/helical/utils.py +0 -51
  147. nabu/pipeline/tests/test_chunk_reader.py +0 -74
  148. {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/LICENSE +0 -0
  149. {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/WHEEL +0 -0
  150. {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/entry_points.txt +0 -0
  151. {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/top_level.txt +0 -0
nabu/io/reader.py CHANGED
@@ -1,12 +1,29 @@
1
1
  import os
2
+ from threading import get_ident
2
3
  from math import ceil
3
4
  from multiprocessing.pool import ThreadPool
5
+ from posixpath import sep as posix_sep, join as posix_join
4
6
  import numpy as np
7
+ from silx.io import get_data
5
8
  from silx.io.dictdump import h5todict
6
9
  from tomoscan.io import HDF5File
7
- from .utils import get_compacted_dataslices, convert_dict_values
10
+ from .utils import get_compacted_dataslices, convert_dict_values, get_first_hdf5_entry
8
11
  from ..misc.binning import binning as image_binning
9
- from ..utils import subsample_dict, get_3D_subregion, get_num_threads
12
+ from ..utils import (
13
+ check_supported,
14
+ deprecated,
15
+ deprecated_class,
16
+ deprecation_warning,
17
+ indices_to_slices,
18
+ compacted_views,
19
+ merge_slices,
20
+ subsample_dict,
21
+ get_3D_subregion,
22
+ get_num_threads,
23
+ get_shape_from_sliced_dims,
24
+ get_size_from_sliced_dimension,
25
+ safe_format,
26
+ )
10
27
 
11
28
  try:
12
29
  from fabio.edfimage import EdfImage
@@ -136,7 +153,7 @@ class EDFReader(Reader):
136
153
  data = self._reader.data
137
154
  else:
138
155
  data = self._reader.fast_read_roi(fname, (slice(self.start_y, self.end_y), slice(self.start_x, self.end_x)))
139
- self._reader.close()
156
+ # self._reader.close()
140
157
  return data
141
158
 
142
159
  def get_data(self, data_url):
@@ -195,9 +212,9 @@ class HDF5Loader:
195
212
  raise ValueError("Please provide either 'data_buffer' or 'pre_allocate'")
196
213
  self.data = data_buffer
197
214
  self._loaded = False
215
+ self.expected_shape = get_hdf5_dataset_shape(fname, data_path, sub_region=sub_region)
198
216
  if pre_allocate:
199
- expected_shape = get_hdf5_dataset_shape(fname, data_path, sub_region=sub_region)
200
- self.data = np.zeros(expected_shape, dtype=dtype)
217
+ self.data = np.zeros(self.expected_shape, dtype=dtype)
201
218
 
202
219
  def _set_subregion(self, sub_region):
203
220
  self.sub_region = sub_region
@@ -211,22 +228,24 @@ class HDF5Loader:
211
228
  self.start_y, self.end_y = None, None
212
229
  self.start_z, self.end_z = None, None
213
230
 
214
- def load_data(self, force_load=False):
231
+ def load_data(self, force_load=False, output=None):
215
232
  if self._loaded and not force_load:
216
233
  return self.data
234
+ output = self.data if output is None else output
217
235
  with HDF5File(self.fname, "r") as fdesc:
218
- if self.data is None:
219
- self.data = fdesc[self.data_path][
236
+ if output is None:
237
+ output = fdesc[self.data_path][
220
238
  self.start_z : self.end_z, self.start_y : self.end_y, self.start_x : self.end_x
221
239
  ]
222
240
  else:
223
- self.data[:] = fdesc[self.data_path][
241
+ output[:] = fdesc[self.data_path][
224
242
  self.start_z : self.end_z, self.start_y : self.end_y, self.start_x : self.end_x
225
243
  ]
226
244
  self._loaded = True
227
- return self.data
245
+ return output
228
246
 
229
247
 
248
+ @deprecated_class("ChunkReader is deprecated since 2024.2.0 and will be removed in a future version", do_print=True)
230
249
  class ChunkReader:
231
250
  """
232
251
  A reader of chunk of images.
@@ -519,6 +538,349 @@ class ChunkReader:
519
538
  return self.files_data
520
539
 
521
540
 
541
+ class VolReaderBase:
542
+ """
543
+ Base class with common code for data readers (EDFStackReader, NXTomoReader, etc)
544
+ """
545
+
546
+ def __init__(self, *args, **kwargs):
547
+ raise ValueError("Base class")
548
+
549
+ def _set_subregion(self, sub_region):
550
+ slice_angle, slice_z, slice_x = (None, None, None)
551
+ if isinstance(sub_region, slice):
552
+ # Assume selection is done only along dimension 0
553
+ slice_angle = sub_region
554
+ slice_z = None
555
+ slice_x = None
556
+ if isinstance(sub_region, (tuple, list)):
557
+ slice_angle, slice_z, slice_x = sub_region
558
+ self.sub_region = (slice_angle or slice(None, None), slice_z or slice(None, None), slice_x or slice(None, None))
559
+
560
+ def _set_processing_function(self, processing_func, processing_func_args, processing_func_kwargs):
561
+ self.processing_func = processing_func
562
+ self._processing_func_args = processing_func_args or []
563
+ self._processing_func_kwargs = processing_func_kwargs or {}
564
+
565
+ def _get_output(self, array):
566
+ if array is not None:
567
+ if array.shape != self.output_shape:
568
+ raise ValueError("Expected output shape %s but got %s" % (self.output_shape, array.shape))
569
+ if array.dtype != self.output_dtype:
570
+ raise ValueError("Expected output dtype '%s' but got '%s'" % (self.output_dtype, array.dtype))
571
+ output = array
572
+ else:
573
+ output = np.zeros(self.output_shape, dtype=self.output_dtype)
574
+ return output
575
+
576
+ def get_frames_indices(self):
577
+ return np.arange(self.data_shape_total[0])[self._source_selection[0]]
578
+
579
+
580
+ class NXTomoReader(VolReaderBase):
581
+ image_key_path = "instrument/detector/image_key_control"
582
+ multiple_frames_per_file = True
583
+
584
+ def __init__(
585
+ self,
586
+ filename,
587
+ data_path="{entry}/instrument/detector/data",
588
+ sub_region=None,
589
+ image_key=0,
590
+ output_dtype=np.float32,
591
+ processing_func=None,
592
+ processing_func_args=None,
593
+ processing_func_kwargs=None,
594
+ ):
595
+ """
596
+ Read a HDF5 file in NXTomo layout.
597
+
598
+ Parameters
599
+ ----------
600
+ filename: str
601
+ Path to the file to read.
602
+ data_path: str
603
+ Path within the HDF5 file, eg. "entry/instrument/detector/data".
604
+ Default is {entry}/data/data where {entry} is a magic keyword for the first entry.
605
+ sub_region: slice or tuple, optional
606
+ Region to select within the data, once the "image key" selection has been done.
607
+ If None, all the data (corresponding to image_key) is selected.
608
+ If slice(start, stop) is provided, the selection is done along dimension 0.
609
+ Otherwise, it must be a 3-tuple of slices in the form
610
+ (slice(start_angle, end_angle, step_angle), slice(start_z, end_z, step_z), slice(start_x, end_x, step_x))
611
+ Each of the parameters can be None, in this case the default start and end are taken in each dimension.
612
+ output_dtype: numpy.dtype, optional
613
+ Output data type if the data memory is allocated by this class. Default is float32.
614
+ image_key: int, or None, optional
615
+ Image type to read (see NXTomo documentation). 0 for projections, 1 for flat-field, 2 for dark field.
616
+ If set to None, all the data will be read.
617
+ processing_func: callable, optional
618
+ Function to be called on each loaded stack of images.
619
+ If provided, this function first argument must be the source buffer (3D array: stack of raw images),
620
+ and the second argument must be the destination buffer (3D array, stack of output images). It can be None.
621
+
622
+ Other parameters
623
+ ----------------
624
+ The other parameters are passed to "processing_func" if this parameter is not None.
625
+
626
+ """
627
+ self.filename = filename
628
+ self.data_path = safe_format(data_path or "", entry=get_first_hdf5_entry(filename))
629
+ self._set_image_key(image_key)
630
+ self._set_subregion(sub_region)
631
+ self._get_shape()
632
+ self._set_processing_function(processing_func, processing_func_args, processing_func_kwargs)
633
+ self._get_source_selection()
634
+ self._init_output(output_dtype)
635
+
636
+ def _get_input_dtype(self):
637
+ return get_hdf5_dataset_dtype(self.filename, self.data_path)
638
+
639
+ def _init_output(self, output_dtype):
640
+ output_shape = get_shape_from_sliced_dims(self.data_shape_total, self._source_selection)
641
+ self.output_dtype = output_dtype
642
+ self._output_shape_no_processing = output_shape
643
+ if self.processing_func is not None:
644
+ test_subvolume = np.zeros((1,) + output_shape[1:], dtype=self._get_input_dtype())
645
+ out = self.processing_func(
646
+ test_subvolume, None, *self._processing_func_args, **self._processing_func_kwargs
647
+ )
648
+ output_image_shape = out.shape[1:]
649
+ output_shape = (output_shape[0],) + output_image_shape
650
+ self.output_shape = output_shape
651
+ self._tmp_dst_buffer = None
652
+
653
+ def _set_image_key(self, image_key):
654
+ self.image_key = image_key
655
+ entry = get_entry_from_h5_path(self.data_path)
656
+ image_key_path = posix_join(entry, self.image_key_path)
657
+ with HDF5File(self.filename, "r") as f:
658
+ image_key_val = f[image_key_path][()]
659
+ idx = np.where(image_key_val == image_key)[0]
660
+ if len(idx) == 0:
661
+ raise FileNotFoundError("No frames found with image key = %d" % image_key)
662
+ self._image_key_slices = indices_to_slices(idx)
663
+
664
+ def _get_shape(self):
665
+ # Shape of the total HDF5-NXTomo data (including darks and flats)
666
+ self.data_shape_total = get_hdf5_dataset_shape(self.filename, self.data_path)
667
+ # Shape of the data once the "image key" is selected
668
+ n_imgs = self.data_shape_total[0]
669
+ self.data_shape_imagekey = (
670
+ sum([get_size_from_sliced_dimension(n_imgs, slice_) for slice_ in self._image_key_slices]),
671
+ ) + self.data_shape_total[1:]
672
+
673
+ # Shape of the data after sub-regions are selected
674
+ self.data_shape_subregion = get_shape_from_sliced_dims(self.data_shape_imagekey, self.sub_region)
675
+
676
+ def _get_source_selection(self):
677
+ if len(self._image_key_slices) == 1 and self.processing_func is None:
678
+ # Simple case:
679
+ # - One single chunk to load, i.e len(self._image_key_slices) == 1
680
+ # - No on-the-fly processing (binning, distortion correction, ...)
681
+ # In this case, we can use h5py read_direct() to avoid extraneous memory consumption
682
+ image_key_slice = self._image_key_slices[0]
683
+ # merge image key selection and user selection (if any)
684
+ self._source_selection = (
685
+ merge_slices(image_key_slice, self.sub_region[0] or slice(None, None)),
686
+ ) + self.sub_region[1:]
687
+ else:
688
+ user_selection_dim0 = self.sub_region[0]
689
+ indices = np.arange(self.data_shape_total[0])
690
+ data_selection_indices_axis0 = np.hstack(
691
+ [indices[image_key_slice][user_selection_dim0] for image_key_slice in self._image_key_slices]
692
+ )
693
+ self._source_selection = (data_selection_indices_axis0,) + self.sub_region[1:]
694
+
695
+ def _get_temporary_buffer(self, convert_after_reading):
696
+ if self._tmp_dst_buffer is None:
697
+ shape = self._output_shape_no_processing
698
+ dtype = self.output_dtype if not (convert_after_reading) else self._get_input_dtype()
699
+ self._tmp_dst_buffer = np.zeros(shape, dtype=dtype)
700
+ return self._tmp_dst_buffer
701
+
702
+ def load_data(self, output=None, convert_after_reading=True):
703
+ """
704
+ Read data.
705
+
706
+ Parameters
707
+ -----------
708
+ output: array-like, optional
709
+ Destination 3D array that will hold the data.
710
+ If provided, use this memory buffer instead of allocating the memory.
711
+ Its shape must be compatible with the selection of 'sub_region' and 'image_key'.
712
+ conver_after_reading: bool, optional
713
+ Whether to do the dtype conversion (if any, eg. uint16 to float32) after data reading.
714
+ With using h5py's read_direct(), reading from uint16 to float32 is extremely slow,
715
+ so data type conversion should be done after reading.
716
+ The drawback is that it requires more memory.
717
+ """
718
+ output = self._get_output(output)
719
+ convert_after_reading &= np.dtype(self.output_dtype) != np.dtype(self._get_input_dtype())
720
+ if convert_after_reading or self.processing_func is not None:
721
+ dst_buffer = self._get_temporary_buffer(convert_after_reading)
722
+ else:
723
+ dst_buffer = output
724
+
725
+ with HDF5File(self.filename, "r") as f:
726
+ dptr = f[self.data_path]
727
+ dptr.read_direct(
728
+ dst_buffer,
729
+ source_sel=self._source_selection,
730
+ dest_sel=None,
731
+ )
732
+ if self.processing_func is not None:
733
+ self.processing_func(dst_buffer, output, *self._processing_func_args, **self._processing_func_kwargs)
734
+ elif dst_buffer.ctypes.data != output.ctypes.data:
735
+ output[:] = dst_buffer[:] # cast
736
+
737
+ return output
738
+
739
+
740
+ class NXDarksFlats:
741
+
742
+ _reduce_func = {
743
+ "median": np.median,
744
+ "mean": np.mean,
745
+ }
746
+
747
+ def __init__(self, filename, **nxtomoreader_kwargs):
748
+ nxtomoreader_kwargs.pop("image_key", None)
749
+ self.darks_reader = NXTomoReader(filename, image_key=2, **nxtomoreader_kwargs)
750
+ self.flats_reader = NXTomoReader(filename, image_key=1, **nxtomoreader_kwargs)
751
+ self._raw_darks = None
752
+ self._raw_flats = None
753
+
754
+ def _get_raw_frames(self, what, force_reload=False, as_multiple_array=True):
755
+ check_supported(what, ["darks", "flats"], "frame type")
756
+ loaded_frames = getattr(self, "_raw_%s" % what)
757
+ reader = getattr(self, "%s_reader" % what)
758
+ if force_reload or loaded_frames is None:
759
+ loaded_frames = reader.load_data()
760
+ setattr(self, "_raw_%s" % what, loaded_frames)
761
+ res = loaded_frames
762
+ if as_multiple_array:
763
+ slices_ = compacted_views(reader._image_key_slices)
764
+ return [res[slice_] for slice_ in slices_]
765
+ return res
766
+
767
+ def _get_reduced_frames(self, what, method="mean", force_reload=False, as_dict=False):
768
+ raw_frames = self._get_raw_frames(what, force_reload=force_reload, as_multiple_array=True)
769
+ reduced_frames = [self._reduce_func[method](frames, axis=0) for frames in raw_frames]
770
+ reader = getattr(self, "%s_reader" % what)
771
+ if as_dict:
772
+ return {k: v for k, v in zip([s.start for s in reader._image_key_slices], reduced_frames)}
773
+ return reduced_frames
774
+
775
+ def get_raw_darks(self, force_reload=False, as_multiple_array=True):
776
+ return self._get_raw_frames("darks", force_reload=force_reload, as_multiple_array=as_multiple_array)
777
+
778
+ def get_raw_flats(self, force_reload=False, as_multiple_array=True):
779
+ return self._get_raw_frames("flats", force_reload=force_reload, as_multiple_array=as_multiple_array)
780
+
781
+ def get_reduced_darks(self, method="mean", force_reload=False, as_dict=False):
782
+ return self._get_reduced_frames("darks", method=method, force_reload=force_reload, as_dict=as_dict)
783
+
784
+ def get_reduced_flats(self, method="median", force_reload=False, as_dict=False):
785
+ return self._get_reduced_frames("flats", method=method, force_reload=force_reload, as_dict=as_dict)
786
+
787
+ def get_raw_current(self, h5_path="{entry}/control/data"):
788
+ h5_path = safe_format(h5_path, entry=self.flats_reader.data_path.split(posix_sep)[0])
789
+ with HDF5File(self.flats_reader.filename, "r") as f:
790
+ current = f[h5_path][()]
791
+ return {sl.start: current[sl] for sl in self.flats_reader._image_key_slices}
792
+
793
+ def get_reduced_current(self, h5_path="{entry}/control/data", method="median"):
794
+ current = self.get_raw_current(h5_path=h5_path)
795
+ return {k: self._reduce_func[method](current[k]) for k in current.keys()}
796
+
797
+
798
+ class EDFStackReader(VolReaderBase):
799
+ multiple_frames_per_file = False
800
+
801
+ def __init__(
802
+ self,
803
+ filenames,
804
+ sub_region=None,
805
+ output_dtype=np.float32,
806
+ n_reading_threads=1,
807
+ processing_func=None,
808
+ processing_func_args=None,
809
+ processing_func_kwargs=None,
810
+ ):
811
+ self.filenames = filenames
812
+ self.n_reading_threads = n_reading_threads
813
+ self._set_subregion(sub_region)
814
+ self._get_shape()
815
+ self._set_processing_function(processing_func, processing_func_args, processing_func_kwargs)
816
+ self._get_source_selection()
817
+ self._init_output(output_dtype)
818
+
819
+ def _get_input_dtype(self):
820
+ return EDFReader().read(self.filenames[0]).dtype
821
+
822
+ def _get_shape(self):
823
+ first_filename = self.filenames[0]
824
+
825
+ # Shape of the total data (without subregion selection)
826
+ reader_all = EDFReader()
827
+ first_frame_full = reader_all.read(first_filename)
828
+ self.data_shape_total = (len(self.filenames),) + first_frame_full.shape
829
+ self.input_dtype = first_frame_full.dtype
830
+
831
+ self.data_shape_subregion = get_shape_from_sliced_dims(
832
+ self.data_shape_total, self.sub_region
833
+ ) # might fail if sub_region is not a slice ?
834
+
835
+ def _init_output(self, output_dtype):
836
+ output_shape = get_shape_from_sliced_dims(self.data_shape_total, self._source_selection)
837
+ self.output_dtype = output_dtype
838
+ if self.processing_func is not None:
839
+ test_image = np.zeros(output_shape[1:], dtype=self._get_input_dtype())
840
+ out = self.processing_func(test_image, *self._processing_func_args, **self._processing_func_kwargs)
841
+ output_image_shape = out.shape
842
+ output_shape = (output_shape[0],) + output_image_shape
843
+ self.output_shape = output_shape
844
+
845
+ def _get_source_selection(self):
846
+ self._source_selection = self.sub_region
847
+
848
+ self._sub_region_xy_for_edf_reader = (
849
+ self.sub_region[2].start or 0,
850
+ self.sub_region[2].stop or self.data_shape_total[2],
851
+ self.sub_region[1].start or 0,
852
+ self.sub_region[1].stop or self.data_shape_total[1],
853
+ )
854
+ self.filenames_subsampled = self.filenames[self.sub_region[0]]
855
+
856
+ def load_data(self, output=None):
857
+ output = self._get_output(output)
858
+
859
+ readers = {}
860
+
861
+ def _init_reader_thread():
862
+ readers[get_ident()] = EDFReader(self._sub_region_xy_for_edf_reader)
863
+
864
+ def _get_data(i_fname):
865
+ i, fname = i_fname
866
+ reader = readers[get_ident()]
867
+ frame = reader.read(fname)
868
+ if self.processing_func is not None:
869
+ frame = self.processing_func(frame, *self._processing_func_args, **self._processing_func_kwargs)
870
+ output[i] = frame
871
+
872
+ with ThreadPool(self.n_reading_threads, initializer=_init_reader_thread) as tp:
873
+ tp.map(
874
+ _get_data,
875
+ zip(
876
+ range(len(self.filenames_subsampled)),
877
+ self.filenames_subsampled,
878
+ ),
879
+ )
880
+
881
+ return output
882
+
883
+
522
884
  Readers = {
523
885
  "edf": EDFReader,
524
886
  "hdf5": HDF5Reader,
@@ -529,7 +891,10 @@ Readers = {
529
891
  }
530
892
 
531
893
 
532
- def load_images_from_dataurl_dict(data_url_dict, **chunk_reader_kwargs):
894
+ @deprecated(
895
+ "Function load_images_from_dataurl_dict is deprecated and will be removed in a fugure version", do_print=True
896
+ )
897
+ def load_images_from_dataurl_dict(data_url_dict, sub_region=None, dtype="f", binning=None):
533
898
  """
534
899
  Load a dictionary of dataurl into numpy arrays.
535
900
 
@@ -538,22 +903,40 @@ def load_images_from_dataurl_dict(data_url_dict, **chunk_reader_kwargs):
538
903
  data_url_dict: dict
539
904
  A dictionary where the keys are integers (the index of each image in the dataset),
540
905
  and the values are numpy.ndarray (data_url_dict).
541
-
542
- Other parameters
543
- -----------------
544
- chunk_reader_kwargs: params
545
- Named parameters passed to `nabu.io.reader.ChunkReader`.
906
+ sub_region: tuple, optional
907
+ Tuple in the form (y_subregion, x_subregion)
908
+ where xy_subregion is a tuple in the form slice(start, stop, step)
546
909
 
547
910
  Returns
548
911
  --------
549
912
  res: dict
550
913
  A dictionary where the keys are the same as `data_url_dict`, and the values are numpy arrays.
914
+
915
+ Notes
916
+ -----
917
+ This function is used to load flats/darks. Usually, these are reduced flats/darks, meaning that
918
+ 'data_url_dict' is a collection of a handful of files (less than 10).
919
+ To load more frames, it would be best to use NXTomoReader / EDFStackReader.
551
920
  """
552
- chunk_reader = ChunkReader(data_url_dict, **chunk_reader_kwargs)
553
- img_dict = {}
554
- for img_idx, img_url in data_url_dict.items():
555
- img_dict[img_idx] = chunk_reader.get_data(img_url)
556
- return img_dict
921
+ res = {}
922
+ if sub_region is not None and not isinstance(sub_region[0], slice):
923
+ if len(sub_region) == 4: # (start_y, end_y, start_x, end_x)
924
+ deprecation_warning(
925
+ "The parameter 'sub_region' was passed as (start_x, end_x, start_y, end_y). This is deprecated and will yield an error in the future. Please use the syntax ((start_z, end_z), (start_x, end_x))",
926
+ do_print=True,
927
+ func_name="load_images_from_dataurl_dict",
928
+ )
929
+ sub_region = (slice(sub_region[2], sub_region[3]), slice(sub_region[0], sub_region[1]))
930
+ else: # ((start_z, end_z), (start_x, end_x))
931
+ sub_region = tuple(slice(s[0], s[1]) for s in sub_region)
932
+ for idx, data_url in data_url_dict.items():
933
+ frame = get_data(data_url)
934
+ if sub_region is not None:
935
+ frame = frame[sub_region[0], sub_region[1]]
936
+ if binning is not None:
937
+ frame = image_binning(frame, binning, out_dtype=dtype)
938
+ res[idx] = frame
939
+ return res
557
940
 
558
941
 
559
942
  def load_images_stack_from_hdf5(fname, h5_data_path, sub_region=None):
@@ -591,6 +974,18 @@ def get_hdf5_dataset_shape(fname, h5_data_path, sub_region=None):
591
974
  return tuple(res_shape)
592
975
 
593
976
 
977
+ def get_hdf5_dataset_dtype(fname, h5_data_path):
978
+ with HDF5File(fname, "r") as f:
979
+ d_ptr = f[h5_data_path]
980
+ dtype = d_ptr.dtype
981
+ return dtype
982
+
983
+
984
+ def get_entry_from_h5_path(h5_path):
985
+ v = h5_path.split(posix_sep)
986
+ return v[0] or v[1]
987
+
988
+
594
989
  def check_virtual_sources_exist(fname, data_path):
595
990
  with HDF5File(fname, "r") as f:
596
991
  if data_path not in f: