nabu 2024.1.9__py3-none-any.whl → 2024.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nabu/__init__.py +1 -1
- nabu/app/bootstrap.py +2 -3
- nabu/app/cast_volume.py +4 -2
- nabu/app/cli_configs.py +5 -0
- nabu/app/composite_cor.py +1 -1
- nabu/app/create_distortion_map_from_poly.py +5 -6
- nabu/app/diag_to_pix.py +7 -19
- nabu/app/diag_to_rot.py +14 -29
- nabu/app/double_flatfield.py +32 -44
- nabu/app/parse_reconstruction_log.py +3 -0
- nabu/app/reconstruct.py +53 -15
- nabu/app/reconstruct_helical.py +2 -2
- nabu/app/stitching.py +27 -13
- nabu/app/tests/test_reduce_dark_flat.py +4 -1
- nabu/cuda/kernel.py +11 -2
- nabu/cuda/processing.py +2 -2
- nabu/cuda/src/cone.cu +77 -0
- nabu/cuda/src/hierarchical_backproj.cu +271 -0
- nabu/cuda/utils.py +0 -6
- nabu/estimation/alignment.py +5 -19
- nabu/estimation/cor.py +173 -599
- nabu/estimation/cor_sino.py +356 -26
- nabu/estimation/focus.py +63 -11
- nabu/estimation/tests/test_cor.py +124 -58
- nabu/estimation/tests/test_focus.py +6 -6
- nabu/estimation/tilt.py +2 -1
- nabu/estimation/utils.py +5 -33
- nabu/io/__init__.py +1 -1
- nabu/io/cast_volume.py +1 -1
- nabu/io/reader.py +416 -21
- nabu/io/tests/test_readers.py +422 -0
- nabu/io/tests/test_writers.py +1 -102
- nabu/io/writer.py +4 -433
- nabu/opencl/kernel.py +14 -3
- nabu/opencl/processing.py +8 -0
- nabu/pipeline/config_validators.py +5 -2
- nabu/pipeline/datadump.py +12 -5
- nabu/pipeline/estimators.py +162 -188
- nabu/pipeline/fullfield/chunked.py +168 -92
- nabu/pipeline/fullfield/chunked_cuda.py +7 -3
- nabu/pipeline/fullfield/computations.py +2 -7
- nabu/pipeline/fullfield/dataset_validator.py +0 -4
- nabu/pipeline/fullfield/nabu_config.py +37 -13
- nabu/pipeline/fullfield/processconfig.py +22 -13
- nabu/pipeline/fullfield/reconstruction.py +13 -9
- nabu/pipeline/helical/helical_chunked_regridded.py +1 -1
- nabu/pipeline/helical/helical_chunked_regridded_cuda.py +1 -0
- nabu/pipeline/helical/helical_reconstruction.py +1 -1
- nabu/pipeline/params.py +21 -1
- nabu/pipeline/processconfig.py +1 -12
- nabu/pipeline/reader.py +146 -0
- nabu/pipeline/tests/test_estimators.py +44 -72
- nabu/pipeline/utils.py +4 -2
- nabu/pipeline/writer.py +10 -2
- nabu/preproc/ccd_cuda.py +1 -1
- nabu/preproc/ctf.py +14 -7
- nabu/preproc/ctf_cuda.py +2 -3
- nabu/preproc/double_flatfield.py +5 -12
- nabu/preproc/double_flatfield_cuda.py +2 -2
- nabu/preproc/flatfield.py +5 -1
- nabu/preproc/flatfield_cuda.py +5 -1
- nabu/preproc/phase.py +24 -73
- nabu/preproc/phase_cuda.py +5 -8
- nabu/preproc/tests/test_ctf.py +11 -7
- nabu/preproc/tests/test_flatfield.py +67 -122
- nabu/preproc/tests/test_paganin.py +54 -30
- nabu/processing/azim.py +206 -0
- nabu/processing/convolution_cuda.py +1 -1
- nabu/processing/fft_cuda.py +15 -17
- nabu/processing/histogram.py +2 -0
- nabu/processing/histogram_cuda.py +2 -1
- nabu/processing/kernel_base.py +3 -0
- nabu/processing/muladd_cuda.py +1 -0
- nabu/processing/padding_opencl.py +1 -1
- nabu/processing/roll_opencl.py +1 -0
- nabu/processing/rotation_cuda.py +2 -2
- nabu/processing/tests/test_fft.py +17 -10
- nabu/processing/unsharp_cuda.py +1 -1
- nabu/reconstruction/cone.py +104 -40
- nabu/reconstruction/fbp.py +3 -0
- nabu/reconstruction/fbp_base.py +7 -2
- nabu/reconstruction/filtering.py +20 -7
- nabu/reconstruction/filtering_cuda.py +7 -1
- nabu/reconstruction/hbp.py +424 -0
- nabu/reconstruction/mlem.py +99 -0
- nabu/reconstruction/reconstructor.py +2 -0
- nabu/reconstruction/rings_cuda.py +19 -19
- nabu/reconstruction/sinogram_cuda.py +1 -0
- nabu/reconstruction/sinogram_opencl.py +3 -1
- nabu/reconstruction/tests/test_cone.py +10 -5
- nabu/reconstruction/tests/test_deringer.py +7 -6
- nabu/reconstruction/tests/test_fbp.py +124 -10
- nabu/reconstruction/tests/test_filtering.py +13 -11
- nabu/reconstruction/tests/test_halftomo.py +30 -4
- nabu/reconstruction/tests/test_mlem.py +91 -0
- nabu/reconstruction/tests/test_reconstructor.py +8 -3
- nabu/resources/dataset_analyzer.py +142 -92
- nabu/resources/gpu.py +1 -0
- nabu/resources/nxflatfield.py +134 -125
- nabu/resources/templates/id16a_fluo.conf +42 -0
- nabu/resources/tests/test_extract.py +10 -0
- nabu/resources/tests/test_nxflatfield.py +2 -2
- nabu/stitching/alignment.py +80 -24
- nabu/stitching/config.py +105 -68
- nabu/stitching/definitions.py +1 -0
- nabu/stitching/frame_composition.py +68 -60
- nabu/stitching/overlap.py +91 -51
- nabu/stitching/single_axis_stitching.py +32 -0
- nabu/stitching/slurm_utils.py +6 -6
- nabu/stitching/stitcher/__init__.py +0 -0
- nabu/stitching/stitcher/base.py +124 -0
- nabu/stitching/stitcher/dumper/__init__.py +3 -0
- nabu/stitching/stitcher/dumper/base.py +94 -0
- nabu/stitching/stitcher/dumper/postprocessing.py +356 -0
- nabu/stitching/stitcher/dumper/preprocessing.py +60 -0
- nabu/stitching/stitcher/post_processing.py +555 -0
- nabu/stitching/stitcher/pre_processing.py +1068 -0
- nabu/stitching/stitcher/single_axis.py +484 -0
- nabu/stitching/stitcher/stitcher.py +0 -0
- nabu/stitching/stitcher/y_stitcher.py +13 -0
- nabu/stitching/stitcher/z_stitcher.py +45 -0
- nabu/stitching/stitcher_2D.py +278 -0
- nabu/stitching/tests/test_config.py +12 -37
- nabu/stitching/tests/test_frame_composition.py +33 -59
- nabu/stitching/tests/test_overlap.py +149 -7
- nabu/stitching/tests/test_utils.py +1 -1
- nabu/stitching/tests/test_y_preprocessing_stitching.py +132 -0
- nabu/stitching/tests/{test_z_stitching.py → test_z_postprocessing_stitching.py} +167 -561
- nabu/stitching/tests/test_z_preprocessing_stitching.py +431 -0
- nabu/stitching/utils/__init__.py +1 -0
- nabu/stitching/utils/post_processing.py +281 -0
- nabu/stitching/utils/tests/test_post-processing.py +21 -0
- nabu/stitching/{utils.py → utils/utils.py} +79 -52
- nabu/stitching/y_stitching.py +27 -0
- nabu/stitching/z_stitching.py +32 -2263
- nabu/testutils.py +1 -152
- nabu/thirdparty/tomocupy_remove_stripe.py +43 -9
- nabu/utils.py +158 -61
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/METADATA +10 -3
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/RECORD +144 -121
- nabu/io/tiffwriter_zmm.py +0 -99
- nabu/pipeline/fallback_utils.py +0 -149
- nabu/pipeline/helical/tests/test_accumulator.py +0 -158
- nabu/pipeline/helical/tests/test_pipeline_elements_full.py +0 -355
- nabu/pipeline/helical/tests/test_strategy.py +0 -61
- nabu/pipeline/helical/utils.py +0 -51
- nabu/pipeline/tests/test_chunk_reader.py +0 -74
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/LICENSE +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/WHEEL +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/entry_points.txt +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/top_level.txt +0 -0
nabu/io/reader.py
CHANGED
@@ -1,12 +1,29 @@
|
|
1
1
|
import os
|
2
|
+
from threading import get_ident
|
2
3
|
from math import ceil
|
3
4
|
from multiprocessing.pool import ThreadPool
|
5
|
+
from posixpath import sep as posix_sep, join as posix_join
|
4
6
|
import numpy as np
|
7
|
+
from silx.io import get_data
|
5
8
|
from silx.io.dictdump import h5todict
|
6
9
|
from tomoscan.io import HDF5File
|
7
|
-
from .utils import get_compacted_dataslices, convert_dict_values
|
10
|
+
from .utils import get_compacted_dataslices, convert_dict_values, get_first_hdf5_entry
|
8
11
|
from ..misc.binning import binning as image_binning
|
9
|
-
from ..utils import
|
12
|
+
from ..utils import (
|
13
|
+
check_supported,
|
14
|
+
deprecated,
|
15
|
+
deprecated_class,
|
16
|
+
deprecation_warning,
|
17
|
+
indices_to_slices,
|
18
|
+
compacted_views,
|
19
|
+
merge_slices,
|
20
|
+
subsample_dict,
|
21
|
+
get_3D_subregion,
|
22
|
+
get_num_threads,
|
23
|
+
get_shape_from_sliced_dims,
|
24
|
+
get_size_from_sliced_dimension,
|
25
|
+
safe_format,
|
26
|
+
)
|
10
27
|
|
11
28
|
try:
|
12
29
|
from fabio.edfimage import EdfImage
|
@@ -136,7 +153,7 @@ class EDFReader(Reader):
|
|
136
153
|
data = self._reader.data
|
137
154
|
else:
|
138
155
|
data = self._reader.fast_read_roi(fname, (slice(self.start_y, self.end_y), slice(self.start_x, self.end_x)))
|
139
|
-
self._reader.close()
|
156
|
+
# self._reader.close()
|
140
157
|
return data
|
141
158
|
|
142
159
|
def get_data(self, data_url):
|
@@ -195,9 +212,9 @@ class HDF5Loader:
|
|
195
212
|
raise ValueError("Please provide either 'data_buffer' or 'pre_allocate'")
|
196
213
|
self.data = data_buffer
|
197
214
|
self._loaded = False
|
215
|
+
self.expected_shape = get_hdf5_dataset_shape(fname, data_path, sub_region=sub_region)
|
198
216
|
if pre_allocate:
|
199
|
-
|
200
|
-
self.data = np.zeros(expected_shape, dtype=dtype)
|
217
|
+
self.data = np.zeros(self.expected_shape, dtype=dtype)
|
201
218
|
|
202
219
|
def _set_subregion(self, sub_region):
|
203
220
|
self.sub_region = sub_region
|
@@ -211,22 +228,24 @@ class HDF5Loader:
|
|
211
228
|
self.start_y, self.end_y = None, None
|
212
229
|
self.start_z, self.end_z = None, None
|
213
230
|
|
214
|
-
def load_data(self, force_load=False):
|
231
|
+
def load_data(self, force_load=False, output=None):
|
215
232
|
if self._loaded and not force_load:
|
216
233
|
return self.data
|
234
|
+
output = self.data if output is None else output
|
217
235
|
with HDF5File(self.fname, "r") as fdesc:
|
218
|
-
if
|
219
|
-
|
236
|
+
if output is None:
|
237
|
+
output = fdesc[self.data_path][
|
220
238
|
self.start_z : self.end_z, self.start_y : self.end_y, self.start_x : self.end_x
|
221
239
|
]
|
222
240
|
else:
|
223
|
-
|
241
|
+
output[:] = fdesc[self.data_path][
|
224
242
|
self.start_z : self.end_z, self.start_y : self.end_y, self.start_x : self.end_x
|
225
243
|
]
|
226
244
|
self._loaded = True
|
227
|
-
return
|
245
|
+
return output
|
228
246
|
|
229
247
|
|
248
|
+
@deprecated_class("ChunkReader is deprecated since 2024.2.0 and will be removed in a future version", do_print=True)
|
230
249
|
class ChunkReader:
|
231
250
|
"""
|
232
251
|
A reader of chunk of images.
|
@@ -519,6 +538,349 @@ class ChunkReader:
|
|
519
538
|
return self.files_data
|
520
539
|
|
521
540
|
|
541
|
+
class VolReaderBase:
|
542
|
+
"""
|
543
|
+
Base class with common code for data readers (EDFStackReader, NXTomoReader, etc)
|
544
|
+
"""
|
545
|
+
|
546
|
+
def __init__(self, *args, **kwargs):
|
547
|
+
raise ValueError("Base class")
|
548
|
+
|
549
|
+
def _set_subregion(self, sub_region):
|
550
|
+
slice_angle, slice_z, slice_x = (None, None, None)
|
551
|
+
if isinstance(sub_region, slice):
|
552
|
+
# Assume selection is done only along dimension 0
|
553
|
+
slice_angle = sub_region
|
554
|
+
slice_z = None
|
555
|
+
slice_x = None
|
556
|
+
if isinstance(sub_region, (tuple, list)):
|
557
|
+
slice_angle, slice_z, slice_x = sub_region
|
558
|
+
self.sub_region = (slice_angle or slice(None, None), slice_z or slice(None, None), slice_x or slice(None, None))
|
559
|
+
|
560
|
+
def _set_processing_function(self, processing_func, processing_func_args, processing_func_kwargs):
|
561
|
+
self.processing_func = processing_func
|
562
|
+
self._processing_func_args = processing_func_args or []
|
563
|
+
self._processing_func_kwargs = processing_func_kwargs or {}
|
564
|
+
|
565
|
+
def _get_output(self, array):
|
566
|
+
if array is not None:
|
567
|
+
if array.shape != self.output_shape:
|
568
|
+
raise ValueError("Expected output shape %s but got %s" % (self.output_shape, array.shape))
|
569
|
+
if array.dtype != self.output_dtype:
|
570
|
+
raise ValueError("Expected output dtype '%s' but got '%s'" % (self.output_dtype, array.dtype))
|
571
|
+
output = array
|
572
|
+
else:
|
573
|
+
output = np.zeros(self.output_shape, dtype=self.output_dtype)
|
574
|
+
return output
|
575
|
+
|
576
|
+
def get_frames_indices(self):
|
577
|
+
return np.arange(self.data_shape_total[0])[self._source_selection[0]]
|
578
|
+
|
579
|
+
|
580
|
+
class NXTomoReader(VolReaderBase):
|
581
|
+
image_key_path = "instrument/detector/image_key_control"
|
582
|
+
multiple_frames_per_file = True
|
583
|
+
|
584
|
+
def __init__(
|
585
|
+
self,
|
586
|
+
filename,
|
587
|
+
data_path="{entry}/instrument/detector/data",
|
588
|
+
sub_region=None,
|
589
|
+
image_key=0,
|
590
|
+
output_dtype=np.float32,
|
591
|
+
processing_func=None,
|
592
|
+
processing_func_args=None,
|
593
|
+
processing_func_kwargs=None,
|
594
|
+
):
|
595
|
+
"""
|
596
|
+
Read a HDF5 file in NXTomo layout.
|
597
|
+
|
598
|
+
Parameters
|
599
|
+
----------
|
600
|
+
filename: str
|
601
|
+
Path to the file to read.
|
602
|
+
data_path: str
|
603
|
+
Path within the HDF5 file, eg. "entry/instrument/detector/data".
|
604
|
+
Default is {entry}/data/data where {entry} is a magic keyword for the first entry.
|
605
|
+
sub_region: slice or tuple, optional
|
606
|
+
Region to select within the data, once the "image key" selection has been done.
|
607
|
+
If None, all the data (corresponding to image_key) is selected.
|
608
|
+
If slice(start, stop) is provided, the selection is done along dimension 0.
|
609
|
+
Otherwise, it must be a 3-tuple of slices in the form
|
610
|
+
(slice(start_angle, end_angle, step_angle), slice(start_z, end_z, step_z), slice(start_x, end_x, step_x))
|
611
|
+
Each of the parameters can be None, in this case the default start and end are taken in each dimension.
|
612
|
+
output_dtype: numpy.dtype, optional
|
613
|
+
Output data type if the data memory is allocated by this class. Default is float32.
|
614
|
+
image_key: int, or None, optional
|
615
|
+
Image type to read (see NXTomo documentation). 0 for projections, 1 for flat-field, 2 for dark field.
|
616
|
+
If set to None, all the data will be read.
|
617
|
+
processing_func: callable, optional
|
618
|
+
Function to be called on each loaded stack of images.
|
619
|
+
If provided, this function first argument must be the source buffer (3D array: stack of raw images),
|
620
|
+
and the second argument must be the destination buffer (3D array, stack of output images). It can be None.
|
621
|
+
|
622
|
+
Other parameters
|
623
|
+
----------------
|
624
|
+
The other parameters are passed to "processing_func" if this parameter is not None.
|
625
|
+
|
626
|
+
"""
|
627
|
+
self.filename = filename
|
628
|
+
self.data_path = safe_format(data_path or "", entry=get_first_hdf5_entry(filename))
|
629
|
+
self._set_image_key(image_key)
|
630
|
+
self._set_subregion(sub_region)
|
631
|
+
self._get_shape()
|
632
|
+
self._set_processing_function(processing_func, processing_func_args, processing_func_kwargs)
|
633
|
+
self._get_source_selection()
|
634
|
+
self._init_output(output_dtype)
|
635
|
+
|
636
|
+
def _get_input_dtype(self):
|
637
|
+
return get_hdf5_dataset_dtype(self.filename, self.data_path)
|
638
|
+
|
639
|
+
def _init_output(self, output_dtype):
|
640
|
+
output_shape = get_shape_from_sliced_dims(self.data_shape_total, self._source_selection)
|
641
|
+
self.output_dtype = output_dtype
|
642
|
+
self._output_shape_no_processing = output_shape
|
643
|
+
if self.processing_func is not None:
|
644
|
+
test_subvolume = np.zeros((1,) + output_shape[1:], dtype=self._get_input_dtype())
|
645
|
+
out = self.processing_func(
|
646
|
+
test_subvolume, None, *self._processing_func_args, **self._processing_func_kwargs
|
647
|
+
)
|
648
|
+
output_image_shape = out.shape[1:]
|
649
|
+
output_shape = (output_shape[0],) + output_image_shape
|
650
|
+
self.output_shape = output_shape
|
651
|
+
self._tmp_dst_buffer = None
|
652
|
+
|
653
|
+
def _set_image_key(self, image_key):
|
654
|
+
self.image_key = image_key
|
655
|
+
entry = get_entry_from_h5_path(self.data_path)
|
656
|
+
image_key_path = posix_join(entry, self.image_key_path)
|
657
|
+
with HDF5File(self.filename, "r") as f:
|
658
|
+
image_key_val = f[image_key_path][()]
|
659
|
+
idx = np.where(image_key_val == image_key)[0]
|
660
|
+
if len(idx) == 0:
|
661
|
+
raise FileNotFoundError("No frames found with image key = %d" % image_key)
|
662
|
+
self._image_key_slices = indices_to_slices(idx)
|
663
|
+
|
664
|
+
def _get_shape(self):
|
665
|
+
# Shape of the total HDF5-NXTomo data (including darks and flats)
|
666
|
+
self.data_shape_total = get_hdf5_dataset_shape(self.filename, self.data_path)
|
667
|
+
# Shape of the data once the "image key" is selected
|
668
|
+
n_imgs = self.data_shape_total[0]
|
669
|
+
self.data_shape_imagekey = (
|
670
|
+
sum([get_size_from_sliced_dimension(n_imgs, slice_) for slice_ in self._image_key_slices]),
|
671
|
+
) + self.data_shape_total[1:]
|
672
|
+
|
673
|
+
# Shape of the data after sub-regions are selected
|
674
|
+
self.data_shape_subregion = get_shape_from_sliced_dims(self.data_shape_imagekey, self.sub_region)
|
675
|
+
|
676
|
+
def _get_source_selection(self):
|
677
|
+
if len(self._image_key_slices) == 1 and self.processing_func is None:
|
678
|
+
# Simple case:
|
679
|
+
# - One single chunk to load, i.e len(self._image_key_slices) == 1
|
680
|
+
# - No on-the-fly processing (binning, distortion correction, ...)
|
681
|
+
# In this case, we can use h5py read_direct() to avoid extraneous memory consumption
|
682
|
+
image_key_slice = self._image_key_slices[0]
|
683
|
+
# merge image key selection and user selection (if any)
|
684
|
+
self._source_selection = (
|
685
|
+
merge_slices(image_key_slice, self.sub_region[0] or slice(None, None)),
|
686
|
+
) + self.sub_region[1:]
|
687
|
+
else:
|
688
|
+
user_selection_dim0 = self.sub_region[0]
|
689
|
+
indices = np.arange(self.data_shape_total[0])
|
690
|
+
data_selection_indices_axis0 = np.hstack(
|
691
|
+
[indices[image_key_slice][user_selection_dim0] for image_key_slice in self._image_key_slices]
|
692
|
+
)
|
693
|
+
self._source_selection = (data_selection_indices_axis0,) + self.sub_region[1:]
|
694
|
+
|
695
|
+
def _get_temporary_buffer(self, convert_after_reading):
|
696
|
+
if self._tmp_dst_buffer is None:
|
697
|
+
shape = self._output_shape_no_processing
|
698
|
+
dtype = self.output_dtype if not (convert_after_reading) else self._get_input_dtype()
|
699
|
+
self._tmp_dst_buffer = np.zeros(shape, dtype=dtype)
|
700
|
+
return self._tmp_dst_buffer
|
701
|
+
|
702
|
+
def load_data(self, output=None, convert_after_reading=True):
|
703
|
+
"""
|
704
|
+
Read data.
|
705
|
+
|
706
|
+
Parameters
|
707
|
+
-----------
|
708
|
+
output: array-like, optional
|
709
|
+
Destination 3D array that will hold the data.
|
710
|
+
If provided, use this memory buffer instead of allocating the memory.
|
711
|
+
Its shape must be compatible with the selection of 'sub_region' and 'image_key'.
|
712
|
+
conver_after_reading: bool, optional
|
713
|
+
Whether to do the dtype conversion (if any, eg. uint16 to float32) after data reading.
|
714
|
+
With using h5py's read_direct(), reading from uint16 to float32 is extremely slow,
|
715
|
+
so data type conversion should be done after reading.
|
716
|
+
The drawback is that it requires more memory.
|
717
|
+
"""
|
718
|
+
output = self._get_output(output)
|
719
|
+
convert_after_reading &= np.dtype(self.output_dtype) != np.dtype(self._get_input_dtype())
|
720
|
+
if convert_after_reading or self.processing_func is not None:
|
721
|
+
dst_buffer = self._get_temporary_buffer(convert_after_reading)
|
722
|
+
else:
|
723
|
+
dst_buffer = output
|
724
|
+
|
725
|
+
with HDF5File(self.filename, "r") as f:
|
726
|
+
dptr = f[self.data_path]
|
727
|
+
dptr.read_direct(
|
728
|
+
dst_buffer,
|
729
|
+
source_sel=self._source_selection,
|
730
|
+
dest_sel=None,
|
731
|
+
)
|
732
|
+
if self.processing_func is not None:
|
733
|
+
self.processing_func(dst_buffer, output, *self._processing_func_args, **self._processing_func_kwargs)
|
734
|
+
elif dst_buffer.ctypes.data != output.ctypes.data:
|
735
|
+
output[:] = dst_buffer[:] # cast
|
736
|
+
|
737
|
+
return output
|
738
|
+
|
739
|
+
|
740
|
+
class NXDarksFlats:
|
741
|
+
|
742
|
+
_reduce_func = {
|
743
|
+
"median": np.median,
|
744
|
+
"mean": np.mean,
|
745
|
+
}
|
746
|
+
|
747
|
+
def __init__(self, filename, **nxtomoreader_kwargs):
|
748
|
+
nxtomoreader_kwargs.pop("image_key", None)
|
749
|
+
self.darks_reader = NXTomoReader(filename, image_key=2, **nxtomoreader_kwargs)
|
750
|
+
self.flats_reader = NXTomoReader(filename, image_key=1, **nxtomoreader_kwargs)
|
751
|
+
self._raw_darks = None
|
752
|
+
self._raw_flats = None
|
753
|
+
|
754
|
+
def _get_raw_frames(self, what, force_reload=False, as_multiple_array=True):
|
755
|
+
check_supported(what, ["darks", "flats"], "frame type")
|
756
|
+
loaded_frames = getattr(self, "_raw_%s" % what)
|
757
|
+
reader = getattr(self, "%s_reader" % what)
|
758
|
+
if force_reload or loaded_frames is None:
|
759
|
+
loaded_frames = reader.load_data()
|
760
|
+
setattr(self, "_raw_%s" % what, loaded_frames)
|
761
|
+
res = loaded_frames
|
762
|
+
if as_multiple_array:
|
763
|
+
slices_ = compacted_views(reader._image_key_slices)
|
764
|
+
return [res[slice_] for slice_ in slices_]
|
765
|
+
return res
|
766
|
+
|
767
|
+
def _get_reduced_frames(self, what, method="mean", force_reload=False, as_dict=False):
|
768
|
+
raw_frames = self._get_raw_frames(what, force_reload=force_reload, as_multiple_array=True)
|
769
|
+
reduced_frames = [self._reduce_func[method](frames, axis=0) for frames in raw_frames]
|
770
|
+
reader = getattr(self, "%s_reader" % what)
|
771
|
+
if as_dict:
|
772
|
+
return {k: v for k, v in zip([s.start for s in reader._image_key_slices], reduced_frames)}
|
773
|
+
return reduced_frames
|
774
|
+
|
775
|
+
def get_raw_darks(self, force_reload=False, as_multiple_array=True):
|
776
|
+
return self._get_raw_frames("darks", force_reload=force_reload, as_multiple_array=as_multiple_array)
|
777
|
+
|
778
|
+
def get_raw_flats(self, force_reload=False, as_multiple_array=True):
|
779
|
+
return self._get_raw_frames("flats", force_reload=force_reload, as_multiple_array=as_multiple_array)
|
780
|
+
|
781
|
+
def get_reduced_darks(self, method="mean", force_reload=False, as_dict=False):
|
782
|
+
return self._get_reduced_frames("darks", method=method, force_reload=force_reload, as_dict=as_dict)
|
783
|
+
|
784
|
+
def get_reduced_flats(self, method="median", force_reload=False, as_dict=False):
|
785
|
+
return self._get_reduced_frames("flats", method=method, force_reload=force_reload, as_dict=as_dict)
|
786
|
+
|
787
|
+
def get_raw_current(self, h5_path="{entry}/control/data"):
|
788
|
+
h5_path = safe_format(h5_path, entry=self.flats_reader.data_path.split(posix_sep)[0])
|
789
|
+
with HDF5File(self.flats_reader.filename, "r") as f:
|
790
|
+
current = f[h5_path][()]
|
791
|
+
return {sl.start: current[sl] for sl in self.flats_reader._image_key_slices}
|
792
|
+
|
793
|
+
def get_reduced_current(self, h5_path="{entry}/control/data", method="median"):
|
794
|
+
current = self.get_raw_current(h5_path=h5_path)
|
795
|
+
return {k: self._reduce_func[method](current[k]) for k in current.keys()}
|
796
|
+
|
797
|
+
|
798
|
+
class EDFStackReader(VolReaderBase):
|
799
|
+
multiple_frames_per_file = False
|
800
|
+
|
801
|
+
def __init__(
|
802
|
+
self,
|
803
|
+
filenames,
|
804
|
+
sub_region=None,
|
805
|
+
output_dtype=np.float32,
|
806
|
+
n_reading_threads=1,
|
807
|
+
processing_func=None,
|
808
|
+
processing_func_args=None,
|
809
|
+
processing_func_kwargs=None,
|
810
|
+
):
|
811
|
+
self.filenames = filenames
|
812
|
+
self.n_reading_threads = n_reading_threads
|
813
|
+
self._set_subregion(sub_region)
|
814
|
+
self._get_shape()
|
815
|
+
self._set_processing_function(processing_func, processing_func_args, processing_func_kwargs)
|
816
|
+
self._get_source_selection()
|
817
|
+
self._init_output(output_dtype)
|
818
|
+
|
819
|
+
def _get_input_dtype(self):
|
820
|
+
return EDFReader().read(self.filenames[0]).dtype
|
821
|
+
|
822
|
+
def _get_shape(self):
|
823
|
+
first_filename = self.filenames[0]
|
824
|
+
|
825
|
+
# Shape of the total data (without subregion selection)
|
826
|
+
reader_all = EDFReader()
|
827
|
+
first_frame_full = reader_all.read(first_filename)
|
828
|
+
self.data_shape_total = (len(self.filenames),) + first_frame_full.shape
|
829
|
+
self.input_dtype = first_frame_full.dtype
|
830
|
+
|
831
|
+
self.data_shape_subregion = get_shape_from_sliced_dims(
|
832
|
+
self.data_shape_total, self.sub_region
|
833
|
+
) # might fail if sub_region is not a slice ?
|
834
|
+
|
835
|
+
def _init_output(self, output_dtype):
|
836
|
+
output_shape = get_shape_from_sliced_dims(self.data_shape_total, self._source_selection)
|
837
|
+
self.output_dtype = output_dtype
|
838
|
+
if self.processing_func is not None:
|
839
|
+
test_image = np.zeros(output_shape[1:], dtype=self._get_input_dtype())
|
840
|
+
out = self.processing_func(test_image, *self._processing_func_args, **self._processing_func_kwargs)
|
841
|
+
output_image_shape = out.shape
|
842
|
+
output_shape = (output_shape[0],) + output_image_shape
|
843
|
+
self.output_shape = output_shape
|
844
|
+
|
845
|
+
def _get_source_selection(self):
|
846
|
+
self._source_selection = self.sub_region
|
847
|
+
|
848
|
+
self._sub_region_xy_for_edf_reader = (
|
849
|
+
self.sub_region[2].start or 0,
|
850
|
+
self.sub_region[2].stop or self.data_shape_total[2],
|
851
|
+
self.sub_region[1].start or 0,
|
852
|
+
self.sub_region[1].stop or self.data_shape_total[1],
|
853
|
+
)
|
854
|
+
self.filenames_subsampled = self.filenames[self.sub_region[0]]
|
855
|
+
|
856
|
+
def load_data(self, output=None):
|
857
|
+
output = self._get_output(output)
|
858
|
+
|
859
|
+
readers = {}
|
860
|
+
|
861
|
+
def _init_reader_thread():
|
862
|
+
readers[get_ident()] = EDFReader(self._sub_region_xy_for_edf_reader)
|
863
|
+
|
864
|
+
def _get_data(i_fname):
|
865
|
+
i, fname = i_fname
|
866
|
+
reader = readers[get_ident()]
|
867
|
+
frame = reader.read(fname)
|
868
|
+
if self.processing_func is not None:
|
869
|
+
frame = self.processing_func(frame, *self._processing_func_args, **self._processing_func_kwargs)
|
870
|
+
output[i] = frame
|
871
|
+
|
872
|
+
with ThreadPool(self.n_reading_threads, initializer=_init_reader_thread) as tp:
|
873
|
+
tp.map(
|
874
|
+
_get_data,
|
875
|
+
zip(
|
876
|
+
range(len(self.filenames_subsampled)),
|
877
|
+
self.filenames_subsampled,
|
878
|
+
),
|
879
|
+
)
|
880
|
+
|
881
|
+
return output
|
882
|
+
|
883
|
+
|
522
884
|
Readers = {
|
523
885
|
"edf": EDFReader,
|
524
886
|
"hdf5": HDF5Reader,
|
@@ -529,7 +891,10 @@ Readers = {
|
|
529
891
|
}
|
530
892
|
|
531
893
|
|
532
|
-
|
894
|
+
@deprecated(
|
895
|
+
"Function load_images_from_dataurl_dict is deprecated and will be removed in a fugure version", do_print=True
|
896
|
+
)
|
897
|
+
def load_images_from_dataurl_dict(data_url_dict, sub_region=None, dtype="f", binning=None):
|
533
898
|
"""
|
534
899
|
Load a dictionary of dataurl into numpy arrays.
|
535
900
|
|
@@ -538,22 +903,40 @@ def load_images_from_dataurl_dict(data_url_dict, **chunk_reader_kwargs):
|
|
538
903
|
data_url_dict: dict
|
539
904
|
A dictionary where the keys are integers (the index of each image in the dataset),
|
540
905
|
and the values are numpy.ndarray (data_url_dict).
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
chunk_reader_kwargs: params
|
545
|
-
Named parameters passed to `nabu.io.reader.ChunkReader`.
|
906
|
+
sub_region: tuple, optional
|
907
|
+
Tuple in the form (y_subregion, x_subregion)
|
908
|
+
where xy_subregion is a tuple in the form slice(start, stop, step)
|
546
909
|
|
547
910
|
Returns
|
548
911
|
--------
|
549
912
|
res: dict
|
550
913
|
A dictionary where the keys are the same as `data_url_dict`, and the values are numpy arrays.
|
914
|
+
|
915
|
+
Notes
|
916
|
+
-----
|
917
|
+
This function is used to load flats/darks. Usually, these are reduced flats/darks, meaning that
|
918
|
+
'data_url_dict' is a collection of a handful of files (less than 10).
|
919
|
+
To load more frames, it would be best to use NXTomoReader / EDFStackReader.
|
551
920
|
"""
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
921
|
+
res = {}
|
922
|
+
if sub_region is not None and not isinstance(sub_region[0], slice):
|
923
|
+
if len(sub_region) == 4: # (start_y, end_y, start_x, end_x)
|
924
|
+
deprecation_warning(
|
925
|
+
"The parameter 'sub_region' was passed as (start_x, end_x, start_y, end_y). This is deprecated and will yield an error in the future. Please use the syntax ((start_z, end_z), (start_x, end_x))",
|
926
|
+
do_print=True,
|
927
|
+
func_name="load_images_from_dataurl_dict",
|
928
|
+
)
|
929
|
+
sub_region = (slice(sub_region[2], sub_region[3]), slice(sub_region[0], sub_region[1]))
|
930
|
+
else: # ((start_z, end_z), (start_x, end_x))
|
931
|
+
sub_region = tuple(slice(s[0], s[1]) for s in sub_region)
|
932
|
+
for idx, data_url in data_url_dict.items():
|
933
|
+
frame = get_data(data_url)
|
934
|
+
if sub_region is not None:
|
935
|
+
frame = frame[sub_region[0], sub_region[1]]
|
936
|
+
if binning is not None:
|
937
|
+
frame = image_binning(frame, binning, out_dtype=dtype)
|
938
|
+
res[idx] = frame
|
939
|
+
return res
|
557
940
|
|
558
941
|
|
559
942
|
def load_images_stack_from_hdf5(fname, h5_data_path, sub_region=None):
|
@@ -591,6 +974,18 @@ def get_hdf5_dataset_shape(fname, h5_data_path, sub_region=None):
|
|
591
974
|
return tuple(res_shape)
|
592
975
|
|
593
976
|
|
977
|
+
def get_hdf5_dataset_dtype(fname, h5_data_path):
|
978
|
+
with HDF5File(fname, "r") as f:
|
979
|
+
d_ptr = f[h5_data_path]
|
980
|
+
dtype = d_ptr.dtype
|
981
|
+
return dtype
|
982
|
+
|
983
|
+
|
984
|
+
def get_entry_from_h5_path(h5_path):
|
985
|
+
v = h5_path.split(posix_sep)
|
986
|
+
return v[0] or v[1]
|
987
|
+
|
988
|
+
|
594
989
|
def check_virtual_sources_exist(fname, data_path):
|
595
990
|
with HDF5File(fname, "r") as f:
|
596
991
|
if data_path not in f:
|