nabu 2024.1.9__py3-none-any.whl → 2024.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nabu/__init__.py +1 -1
- nabu/app/bootstrap.py +2 -3
- nabu/app/cast_volume.py +4 -2
- nabu/app/cli_configs.py +5 -0
- nabu/app/composite_cor.py +1 -1
- nabu/app/create_distortion_map_from_poly.py +5 -6
- nabu/app/diag_to_pix.py +7 -19
- nabu/app/diag_to_rot.py +14 -29
- nabu/app/double_flatfield.py +32 -44
- nabu/app/parse_reconstruction_log.py +3 -0
- nabu/app/reconstruct.py +53 -15
- nabu/app/reconstruct_helical.py +2 -2
- nabu/app/stitching.py +27 -13
- nabu/app/tests/test_reduce_dark_flat.py +4 -1
- nabu/cuda/kernel.py +11 -2
- nabu/cuda/processing.py +2 -2
- nabu/cuda/src/cone.cu +77 -0
- nabu/cuda/src/hierarchical_backproj.cu +271 -0
- nabu/cuda/utils.py +0 -6
- nabu/estimation/alignment.py +5 -19
- nabu/estimation/cor.py +173 -599
- nabu/estimation/cor_sino.py +356 -26
- nabu/estimation/focus.py +63 -11
- nabu/estimation/tests/test_cor.py +124 -58
- nabu/estimation/tests/test_focus.py +6 -6
- nabu/estimation/tilt.py +2 -1
- nabu/estimation/utils.py +5 -33
- nabu/io/__init__.py +1 -1
- nabu/io/cast_volume.py +1 -1
- nabu/io/reader.py +416 -21
- nabu/io/tests/test_readers.py +422 -0
- nabu/io/tests/test_writers.py +1 -102
- nabu/io/writer.py +4 -433
- nabu/opencl/kernel.py +14 -3
- nabu/opencl/processing.py +8 -0
- nabu/pipeline/config_validators.py +5 -2
- nabu/pipeline/datadump.py +12 -5
- nabu/pipeline/estimators.py +162 -188
- nabu/pipeline/fullfield/chunked.py +168 -92
- nabu/pipeline/fullfield/chunked_cuda.py +7 -3
- nabu/pipeline/fullfield/computations.py +2 -7
- nabu/pipeline/fullfield/dataset_validator.py +0 -4
- nabu/pipeline/fullfield/nabu_config.py +37 -13
- nabu/pipeline/fullfield/processconfig.py +22 -13
- nabu/pipeline/fullfield/reconstruction.py +13 -9
- nabu/pipeline/helical/helical_chunked_regridded.py +1 -1
- nabu/pipeline/helical/helical_chunked_regridded_cuda.py +1 -0
- nabu/pipeline/helical/helical_reconstruction.py +1 -1
- nabu/pipeline/params.py +21 -1
- nabu/pipeline/processconfig.py +1 -12
- nabu/pipeline/reader.py +146 -0
- nabu/pipeline/tests/test_estimators.py +44 -72
- nabu/pipeline/utils.py +4 -2
- nabu/pipeline/writer.py +10 -2
- nabu/preproc/ccd_cuda.py +1 -1
- nabu/preproc/ctf.py +14 -7
- nabu/preproc/ctf_cuda.py +2 -3
- nabu/preproc/double_flatfield.py +5 -12
- nabu/preproc/double_flatfield_cuda.py +2 -2
- nabu/preproc/flatfield.py +5 -1
- nabu/preproc/flatfield_cuda.py +5 -1
- nabu/preproc/phase.py +24 -73
- nabu/preproc/phase_cuda.py +5 -8
- nabu/preproc/tests/test_ctf.py +11 -7
- nabu/preproc/tests/test_flatfield.py +67 -122
- nabu/preproc/tests/test_paganin.py +54 -30
- nabu/processing/azim.py +206 -0
- nabu/processing/convolution_cuda.py +1 -1
- nabu/processing/fft_cuda.py +15 -17
- nabu/processing/histogram.py +2 -0
- nabu/processing/histogram_cuda.py +2 -1
- nabu/processing/kernel_base.py +3 -0
- nabu/processing/muladd_cuda.py +1 -0
- nabu/processing/padding_opencl.py +1 -1
- nabu/processing/roll_opencl.py +1 -0
- nabu/processing/rotation_cuda.py +2 -2
- nabu/processing/tests/test_fft.py +17 -10
- nabu/processing/unsharp_cuda.py +1 -1
- nabu/reconstruction/cone.py +104 -40
- nabu/reconstruction/fbp.py +3 -0
- nabu/reconstruction/fbp_base.py +7 -2
- nabu/reconstruction/filtering.py +20 -7
- nabu/reconstruction/filtering_cuda.py +7 -1
- nabu/reconstruction/hbp.py +424 -0
- nabu/reconstruction/mlem.py +99 -0
- nabu/reconstruction/reconstructor.py +2 -0
- nabu/reconstruction/rings_cuda.py +19 -19
- nabu/reconstruction/sinogram_cuda.py +1 -0
- nabu/reconstruction/sinogram_opencl.py +3 -1
- nabu/reconstruction/tests/test_cone.py +10 -5
- nabu/reconstruction/tests/test_deringer.py +7 -6
- nabu/reconstruction/tests/test_fbp.py +124 -10
- nabu/reconstruction/tests/test_filtering.py +13 -11
- nabu/reconstruction/tests/test_halftomo.py +30 -4
- nabu/reconstruction/tests/test_mlem.py +91 -0
- nabu/reconstruction/tests/test_reconstructor.py +8 -3
- nabu/resources/dataset_analyzer.py +142 -92
- nabu/resources/gpu.py +1 -0
- nabu/resources/nxflatfield.py +134 -125
- nabu/resources/templates/id16a_fluo.conf +42 -0
- nabu/resources/tests/test_extract.py +10 -0
- nabu/resources/tests/test_nxflatfield.py +2 -2
- nabu/stitching/alignment.py +80 -24
- nabu/stitching/config.py +105 -68
- nabu/stitching/definitions.py +1 -0
- nabu/stitching/frame_composition.py +68 -60
- nabu/stitching/overlap.py +91 -51
- nabu/stitching/single_axis_stitching.py +32 -0
- nabu/stitching/slurm_utils.py +6 -6
- nabu/stitching/stitcher/__init__.py +0 -0
- nabu/stitching/stitcher/base.py +124 -0
- nabu/stitching/stitcher/dumper/__init__.py +3 -0
- nabu/stitching/stitcher/dumper/base.py +94 -0
- nabu/stitching/stitcher/dumper/postprocessing.py +356 -0
- nabu/stitching/stitcher/dumper/preprocessing.py +60 -0
- nabu/stitching/stitcher/post_processing.py +555 -0
- nabu/stitching/stitcher/pre_processing.py +1068 -0
- nabu/stitching/stitcher/single_axis.py +484 -0
- nabu/stitching/stitcher/stitcher.py +0 -0
- nabu/stitching/stitcher/y_stitcher.py +13 -0
- nabu/stitching/stitcher/z_stitcher.py +45 -0
- nabu/stitching/stitcher_2D.py +278 -0
- nabu/stitching/tests/test_config.py +12 -37
- nabu/stitching/tests/test_frame_composition.py +33 -59
- nabu/stitching/tests/test_overlap.py +149 -7
- nabu/stitching/tests/test_utils.py +1 -1
- nabu/stitching/tests/test_y_preprocessing_stitching.py +132 -0
- nabu/stitching/tests/{test_z_stitching.py → test_z_postprocessing_stitching.py} +167 -561
- nabu/stitching/tests/test_z_preprocessing_stitching.py +431 -0
- nabu/stitching/utils/__init__.py +1 -0
- nabu/stitching/utils/post_processing.py +281 -0
- nabu/stitching/utils/tests/test_post-processing.py +21 -0
- nabu/stitching/{utils.py → utils/utils.py} +79 -52
- nabu/stitching/y_stitching.py +27 -0
- nabu/stitching/z_stitching.py +32 -2263
- nabu/testutils.py +1 -152
- nabu/thirdparty/tomocupy_remove_stripe.py +43 -9
- nabu/utils.py +158 -61
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/METADATA +10 -3
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/RECORD +144 -121
- nabu/io/tiffwriter_zmm.py +0 -99
- nabu/pipeline/fallback_utils.py +0 -149
- nabu/pipeline/helical/tests/test_accumulator.py +0 -158
- nabu/pipeline/helical/tests/test_pipeline_elements_full.py +0 -355
- nabu/pipeline/helical/tests/test_strategy.py +0 -61
- nabu/pipeline/helical/utils.py +0 -51
- nabu/pipeline/tests/test_chunk_reader.py +0 -74
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/LICENSE +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/WHEEL +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/entry_points.txt +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/top_level.txt +0 -0
@@ -1,13 +1,14 @@
|
|
1
1
|
import os
|
2
|
-
from bisect import bisect_left
|
3
2
|
import numpy as np
|
4
|
-
from silx.io import get_data
|
5
3
|
from silx.io.url import DataUrl
|
4
|
+
from silx.io import get_data
|
6
5
|
from tomoscan.esrf.scan.edfscan import EDFTomoScan
|
7
6
|
from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
|
8
|
-
|
7
|
+
|
8
|
+
from ..utils import check_supported, indices_to_slices
|
9
|
+
from ..io.reader import EDFStackReader, NXDarksFlats, NXTomoReader
|
9
10
|
from ..io.utils import get_compacted_dataslices
|
10
|
-
from .utils import
|
11
|
+
from .utils import get_values_from_file, is_hdf5_extension
|
11
12
|
from .logger import LoggerOrPrint
|
12
13
|
|
13
14
|
from ..pipeline.utils import nabu_env_settings
|
@@ -51,7 +52,7 @@ class DatasetAnalyzer:
|
|
51
52
|
"output_dir": None,
|
52
53
|
"exclude_projections": None,
|
53
54
|
"hdf5_entry": None,
|
54
|
-
"nx_version": 1.0,
|
55
|
+
# "nx_version": 1.0,
|
55
56
|
}
|
56
57
|
# --
|
57
58
|
advanced_options.update(extra_options)
|
@@ -59,16 +60,18 @@ class DatasetAnalyzer:
|
|
59
60
|
|
60
61
|
# pylint: disable=E1136
|
61
62
|
def _get_excluded_projections(self):
|
62
|
-
self._ignore_projections_indices = None
|
63
|
-
self._need_rebuild_tomoscan_object_to_exclude_projections = False
|
64
63
|
excluded_projs = self.extra_options["exclude_projections"]
|
64
|
+
self._ignore_projections = None
|
65
65
|
if excluded_projs is None:
|
66
66
|
return
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
67
|
+
|
68
|
+
if excluded_projs["type"] == "angular_range":
|
69
|
+
excluded_projs["type"] = "range" # compat with tomoscan #pylint: disable=E1137
|
70
|
+
values = excluded_projs["range"]
|
71
|
+
for ignore_kind, dtype in {"indices": np.int32, "angles": np.float32}.items():
|
72
|
+
if excluded_projs["type"] == ignore_kind:
|
73
|
+
values = get_values_from_file(excluded_projs["file"], any_size=True).astype(dtype).tolist()
|
74
|
+
self._ignore_projections = {"kind": excluded_projs["type"], "values": values} # pylint: disable=E0606
|
72
75
|
|
73
76
|
def _init_dataset_scan(self, **kwargs):
|
74
77
|
if self._scanner is None:
|
@@ -83,40 +86,11 @@ class DatasetAnalyzer:
|
|
83
86
|
kwargs["n_frames"] = 1
|
84
87
|
|
85
88
|
self.dataset_scanner = self._scanner( # pylint: disable=E1102
|
86
|
-
self.location, ignore_projections=self.
|
89
|
+
self.location, ignore_projections=self._ignore_projections, **kwargs
|
87
90
|
)
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
if self._need_rebuild_tomoscan_object_to_exclude_projections:
|
92
|
-
# pylint: disable=E1136
|
93
|
-
exclude_projs = self.extra_options["exclude_projections"]
|
94
|
-
rot_angles_deg = np.rad2deg(self.rotation_angles)
|
95
|
-
self._rotation_angles = None # prevent caching
|
96
|
-
# tomoscan only supports ignore_projections=<list of integers>
|
97
|
-
# However this is cumbersome to use, it's more convenient to use angular range or list of angles
|
98
|
-
# But having angles instead of indices implies to already have information on current scan angular range
|
99
|
-
ignore_projections_indices = []
|
100
|
-
if exclude_projs["type"] == "angular_range":
|
101
|
-
exclude_angle_min, exclude_angle_max = exclude_projs["range"]
|
102
|
-
projections_indices = np.array(sorted(self.dataset_scanner.projections.keys()))
|
103
|
-
for proj_idx, angle in zip(projections_indices, rot_angles_deg):
|
104
|
-
if exclude_angle_min <= angle and angle <= exclude_angle_max:
|
105
|
-
ignore_projections_indices.append(proj_idx)
|
106
|
-
elif exclude_projs["type"] == "angles":
|
107
|
-
excluded_angles = get_values_from_file(exclude_projs["file"], any_size=True).astype(np.float32).tolist()
|
108
|
-
for excluded_angle in excluded_angles:
|
109
|
-
proj_idx = bisect_left(rot_angles_deg, excluded_angle)
|
110
|
-
if proj_idx < rot_angles_deg.size:
|
111
|
-
ignore_projections_indices.append(proj_idx)
|
112
|
-
# Rebuild the dataset_scanner instance
|
113
|
-
self._ignore_projections_indices = ignore_projections_indices
|
114
|
-
self.dataset_scanner = self._scanner( # pylint: disable=E1102
|
115
|
-
self.location, ignore_projections=self._ignore_projections_indices, **kwargs
|
116
|
-
)
|
117
|
-
# ---
|
118
|
-
if self._ignore_projections_indices is not None:
|
119
|
-
self.logger.info("Excluding projections: %s" % str(self._ignore_projections_indices))
|
91
|
+
|
92
|
+
if self._ignore_projections is not None:
|
93
|
+
self.logger.info("Excluding projections: %s" % str(self._ignore_projections))
|
120
94
|
|
121
95
|
if nabu_env_settings.skip_tomoscan_checks:
|
122
96
|
self.logger.warning(
|
@@ -124,9 +98,8 @@ class DatasetAnalyzer:
|
|
124
98
|
)
|
125
99
|
self.dataset_scanner.set_check_behavior(run_check=False, raise_error=False)
|
126
100
|
|
127
|
-
self.
|
128
|
-
self.
|
129
|
-
self.darks = self.dataset_scanner.darks
|
101
|
+
self.raw_flats = self.dataset_scanner.flats
|
102
|
+
self.raw_darks = self.dataset_scanner.darks
|
130
103
|
self.n_angles = len(self.dataset_scanner.projections)
|
131
104
|
self.radio_dims = (self.dataset_scanner.dim_1, self.dataset_scanner.dim_2)
|
132
105
|
self._radio_dims_notbinned = self.radio_dims # COMPAT
|
@@ -146,7 +119,10 @@ class DatasetAnalyzer:
|
|
146
119
|
self._pixel_size = None
|
147
120
|
self._distance = None
|
148
121
|
self._flats_srcurrent = None
|
122
|
+
self._projections = None
|
149
123
|
self._projections_srcurrent = None
|
124
|
+
self._reduced_flats = None
|
125
|
+
self._reduced_darks = None
|
150
126
|
|
151
127
|
@property
|
152
128
|
def energy(self):
|
@@ -223,15 +199,19 @@ class DatasetAnalyzer:
|
|
223
199
|
def detector_tilt(self, tilt):
|
224
200
|
self._detector_tilt = tilt
|
225
201
|
|
226
|
-
def _get_srcurrent(self,
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
return
|
202
|
+
def _get_srcurrent(self, frame_type):
|
203
|
+
# To be implemented by inheriting class
|
204
|
+
return None
|
205
|
+
|
206
|
+
@property
|
207
|
+
def projections(self):
|
208
|
+
if self._projections is None:
|
209
|
+
self._projections = self.dataset_scanner.projections
|
210
|
+
return self._projections
|
211
|
+
|
212
|
+
@projections.setter
|
213
|
+
def projections(self, val):
|
214
|
+
raise ValueError
|
235
215
|
|
236
216
|
@property
|
237
217
|
def projections_srcurrent(self):
|
@@ -239,8 +219,7 @@ class DatasetAnalyzer:
|
|
239
219
|
Return the synchrotron electric current for each projection.
|
240
220
|
"""
|
241
221
|
if self._projections_srcurrent is None:
|
242
|
-
|
243
|
-
self._projections_srcurrent = self._get_srcurrent(projections_indices)
|
222
|
+
self._projections_srcurrent = self._get_srcurrent("radios") # pylint: disable=E1128
|
244
223
|
return self._projections_srcurrent
|
245
224
|
|
246
225
|
@projections_srcurrent.setter
|
@@ -253,8 +232,7 @@ class DatasetAnalyzer:
|
|
253
232
|
Return the synchrotron electric current for each flat image.
|
254
233
|
"""
|
255
234
|
if self._flats_srcurrent is None:
|
256
|
-
|
257
|
-
self._flats_srcurrent = self._get_srcurrent(flats_indices)
|
235
|
+
self._flats_srcurrent = self._get_srcurrent("flats") # pylint: disable=E1128
|
258
236
|
return self._flats_srcurrent
|
259
237
|
|
260
238
|
@flats_srcurrent.setter
|
@@ -268,6 +246,32 @@ class DatasetAnalyzer:
|
|
268
246
|
if getattr(self, name, None) is None:
|
269
247
|
raise ValueError(error_msg or str("No information on %s was found in the dataset" % name))
|
270
248
|
|
249
|
+
@property
|
250
|
+
def flats(self):
|
251
|
+
"""
|
252
|
+
Return the REDUCED flat-field images. Either by reducing (median) the raw flats, or a user-defined reduced flats.
|
253
|
+
"""
|
254
|
+
if self._reduced_flats is None:
|
255
|
+
self._reduced_flats = self.get_reduced_flats()
|
256
|
+
return self._reduced_flats
|
257
|
+
|
258
|
+
@flats.setter
|
259
|
+
def flats(self, val):
|
260
|
+
self._reduced_flats = val
|
261
|
+
|
262
|
+
@property
|
263
|
+
def darks(self):
|
264
|
+
"""
|
265
|
+
Return the REDUCED flat-field images. Either by reducing (mean) the raw darks, or a user-defined reduced darks.
|
266
|
+
"""
|
267
|
+
if self._reduced_darks is None:
|
268
|
+
self._reduced_darks = self.get_reduced_darks()
|
269
|
+
return self._reduced_darks
|
270
|
+
|
271
|
+
@darks.setter
|
272
|
+
def darks(self, val):
|
273
|
+
self._reduced_darks = val
|
274
|
+
|
271
275
|
|
272
276
|
class EDFDatasetAnalyzer(DatasetAnalyzer):
|
273
277
|
"""
|
@@ -278,23 +282,7 @@ class EDFDatasetAnalyzer(DatasetAnalyzer):
|
|
278
282
|
kind = "edf"
|
279
283
|
|
280
284
|
def _finish_init(self):
|
281
|
-
|
282
|
-
|
283
|
-
def remove_unused_radios(self):
|
284
|
-
"""
|
285
|
-
Remove "unused" radios.
|
286
|
-
This is used for legacy ESRF scans.
|
287
|
-
"""
|
288
|
-
# Extraneous projections are assumed to be on the end
|
289
|
-
projs_indices = sorted(self.projections.keys())
|
290
|
-
used_radios_range = range(projs_indices[0], len(self.projections))
|
291
|
-
radios_not_used = []
|
292
|
-
for idx in self.projections.keys():
|
293
|
-
if idx not in used_radios_range:
|
294
|
-
radios_not_used.append(idx)
|
295
|
-
for idx in radios_not_used:
|
296
|
-
self.projections.pop(idx)
|
297
|
-
return radios_not_used
|
285
|
+
pass
|
298
286
|
|
299
287
|
def _get_flats_darks(self):
|
300
288
|
return
|
@@ -311,13 +299,34 @@ class EDFDatasetAnalyzer(DatasetAnalyzer):
|
|
311
299
|
return None
|
312
300
|
|
313
301
|
def _get_rotation_angles(self):
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
302
|
+
return np.deg2rad(self.dataset_scanner.rotation_angle())
|
303
|
+
|
304
|
+
def get_reduced_flats(self, **reader_kwargs):
|
305
|
+
if self.raw_flats in [None, {}]:
|
306
|
+
raise FileNotFoundError("No reduced flat ('refHST') found in %s" % self.location)
|
307
|
+
# A few notes:
|
308
|
+
# (1) In principle we could do the reduction (mean/median) from raw frames (ref_xxxx_yyyy)
|
309
|
+
# but for legacy datasets it's always already done (by fasttomo3), and EDF support is supposed to be dropped on our side
|
310
|
+
# (2) We use EDFStackReader class to handle the possible additional data modifications
|
311
|
+
# (eg. subsampling, binning, distortion correction...)
|
312
|
+
# (3) The following spawns one reader instance per file, which is not elegant,
|
313
|
+
# but in principle there are typically 1-2 reduced flats in a scan
|
314
|
+
readers = {k: EDFStackReader([self.raw_flats[k].file_path()], **reader_kwargs) for k in self.raw_flats.keys()}
|
315
|
+
return {k: readers[k].load_data()[0] for k in self.raw_flats.keys()}
|
316
|
+
|
317
|
+
def get_reduced_darks(self, **reader_kwargs):
|
318
|
+
# See notes in get_reduced_flats() above
|
319
|
+
if self.raw_darks in [None, {}]:
|
320
|
+
raise FileNotFoundError("No reduced dark ('darkend.edf' or 'dark.edf') found in %s" % self.location)
|
321
|
+
readers = {k: EDFStackReader([self.raw_darks[k].file_path()], **reader_kwargs) for k in self.raw_darks.keys()}
|
322
|
+
return {k: readers[k].load_data()[0] for k in self.raw_darks.keys()}
|
323
|
+
|
324
|
+
@property
|
325
|
+
def files(self):
|
326
|
+
return sorted([u.file_path() for u in self.dataset_scanner.projections.values()])
|
327
|
+
|
328
|
+
def get_reader(self, **kwargs):
|
329
|
+
return EDFStackReader(self.files, **kwargs)
|
321
330
|
|
322
331
|
|
323
332
|
class HDF5DatasetAnalyzer(DatasetAnalyzer):
|
@@ -326,7 +335,10 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
|
|
326
335
|
"""
|
327
336
|
|
328
337
|
_scanner = NXtomoScan
|
329
|
-
kind = "
|
338
|
+
kind = "nx"
|
339
|
+
# We could import the 1000+ LoC nxtomo.nxobject.nxdetector.ImageKey... or we can do this
|
340
|
+
_image_key_value = {"flats": 1, "darks": 2, "radios": 0}
|
341
|
+
#
|
330
342
|
|
331
343
|
@property
|
332
344
|
def z_translation(self):
|
@@ -353,10 +365,10 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
|
|
353
365
|
def _get_dataset_hdf5_url(self):
|
354
366
|
if len(self.projections) > 0:
|
355
367
|
frames_to_take = self.projections
|
356
|
-
elif len(self.
|
357
|
-
frames_to_take = self.
|
358
|
-
elif len(self.
|
359
|
-
frames_to_take = self.
|
368
|
+
elif len(self.raw_flats) > 0:
|
369
|
+
frames_to_take = self.raw_flats
|
370
|
+
elif len(self.raw_darks) > 0:
|
371
|
+
frames_to_take = self.raw_darks
|
360
372
|
else:
|
361
373
|
raise ValueError("No projections, no flats and no darks ?!")
|
362
374
|
first_proj_idx = sorted(frames_to_take.keys())[0]
|
@@ -397,8 +409,13 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
|
|
397
409
|
slices: list of slice
|
398
410
|
A list where each item is a slice.
|
399
411
|
"""
|
400
|
-
|
401
|
-
|
412
|
+
name_to_attr = {
|
413
|
+
"projections": self.projections,
|
414
|
+
"flats": self.raw_flats,
|
415
|
+
"darks": self.raw_darks,
|
416
|
+
}
|
417
|
+
check_supported(what, name_to_attr.keys(), "image type")
|
418
|
+
images = name_to_attr[what] # dict
|
402
419
|
# we can't directly use set() on slice() object (unhashable). Use tuples
|
403
420
|
slices = set()
|
404
421
|
for du in get_compacted_dataslices(images).values():
|
@@ -410,6 +427,39 @@ class HDF5DatasetAnalyzer(DatasetAnalyzer):
|
|
410
427
|
slices_list = [slice(item[0], item[1]) if item is not None else None for item in list(slices)]
|
411
428
|
return slices_list
|
412
429
|
|
430
|
+
def _select_according_to_frame_type(self, data, frame_type):
|
431
|
+
if data is None:
|
432
|
+
return None
|
433
|
+
return data[self.dataset_scanner.image_key_control == self._image_key_value[frame_type]]
|
434
|
+
|
435
|
+
def get_reduced_flats(self, method="median", force_reload=False, **reader_kwargs):
|
436
|
+
dkrf_reader = NXDarksFlats(
|
437
|
+
self.dataset_hdf5_url.file_path(), data_path=self.dataset_hdf5_url.data_path(), **reader_kwargs
|
438
|
+
)
|
439
|
+
return dkrf_reader.get_reduced_flats(method=method, force_reload=force_reload, as_dict=True)
|
440
|
+
|
441
|
+
def get_reduced_darks(self, method="mean", force_reload=False, **reader_kwargs):
|
442
|
+
dkrf_reader = NXDarksFlats(
|
443
|
+
self.dataset_hdf5_url.file_path(), data_path=self.dataset_hdf5_url.data_path(), **reader_kwargs
|
444
|
+
)
|
445
|
+
return dkrf_reader.get_reduced_darks(method=method, force_reload=force_reload, as_dict=True)
|
446
|
+
|
447
|
+
def _get_srcurrent(self, frame_type):
|
448
|
+
return self._select_according_to_frame_type(self.dataset_scanner.electric_current, frame_type)
|
449
|
+
|
450
|
+
def frames_slices(self, frame_type):
|
451
|
+
"""
|
452
|
+
Return a list of slice objects corresponding to the data corresponding to "frame_type".
|
453
|
+
For example, if the dataset flats are located at indices [1, 2, ..., 99], then
|
454
|
+
frame_slices("flats") will return [slice(0, 100)].
|
455
|
+
"""
|
456
|
+
return indices_to_slices(
|
457
|
+
np.where(self.dataset_scanner.image_key_control == self._image_key_value[frame_type])[0]
|
458
|
+
)
|
459
|
+
|
460
|
+
def get_reader(self, **kwargs):
|
461
|
+
return NXTomoReader(self.dataset_hdf5_url.file_path(), data_path=self.dataset_hdf5_url.data_path(), **kwargs)
|
462
|
+
|
413
463
|
|
414
464
|
def analyze_dataset(dataset_path, extra_options=None, logger=None):
|
415
465
|
if not (os.path.isdir(dataset_path)):
|
nabu/resources/gpu.py
CHANGED
nabu/resources/nxflatfield.py
CHANGED
@@ -1,14 +1,16 @@
|
|
1
1
|
import os
|
2
2
|
import numpy as np
|
3
|
+
from nxtomo.io import HDF5File
|
3
4
|
from silx.io.url import DataUrl
|
4
|
-
from
|
5
|
+
from silx.io import get_data
|
6
|
+
from tomoscan.framereducer.reducedframesinfos import ReducedFramesInfos
|
5
7
|
from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
|
6
8
|
from ..utils import check_supported, is_writeable
|
7
9
|
|
8
10
|
|
9
|
-
def get_frame_possible_urls(dataset_info, user_dir, output_dir
|
11
|
+
def get_frame_possible_urls(dataset_info, user_dir, output_dir):
|
10
12
|
"""
|
11
|
-
Return a
|
13
|
+
Return a dict with the possible location of reduced dark/flat frames.
|
12
14
|
|
13
15
|
Parameters
|
14
16
|
----------
|
@@ -18,19 +20,21 @@ def get_frame_possible_urls(dataset_info, user_dir, output_dir, frame_type):
|
|
18
20
|
User-provided directory location for the reduced frames.
|
19
21
|
output_dir: str or None
|
20
22
|
Output processing directory
|
21
|
-
frame_type: str
|
22
|
-
Frame type, can be "flats" or "darks".
|
23
23
|
"""
|
24
|
-
check_supported(frame_type, ["flats", "darks"], "frame type")
|
25
24
|
|
25
|
+
frame_types = ["flats", "darks"]
|
26
26
|
h5scan = dataset_info.dataset_scanner # tomoscan object
|
27
|
-
if frame_type == "flats":
|
28
|
-
dataurl_default_template = h5scan.REDUCED_FLATS_DATAURLS[0]
|
29
|
-
else:
|
30
|
-
dataurl_default_template = h5scan.REDUCED_DARKS_DATAURLS[0]
|
31
27
|
|
32
|
-
def make_dataurl(dirname):
|
33
|
-
|
28
|
+
def make_dataurl(dirname, frame_type):
|
29
|
+
"""
|
30
|
+
The template formatting should be done by tomoscan in principle, but this complicates logging.
|
31
|
+
"""
|
32
|
+
|
33
|
+
if frame_type == "flats":
|
34
|
+
dataurl_default_template = h5scan.REDUCED_FLATS_DATAURLS[0]
|
35
|
+
else:
|
36
|
+
dataurl_default_template = h5scan.REDUCED_DARKS_DATAURLS[0]
|
37
|
+
|
34
38
|
rel_file_path = dataurl_default_template.file_path().format(
|
35
39
|
scan_prefix=dataset_info.dataset_scanner.get_dataset_basename()
|
36
40
|
)
|
@@ -44,18 +48,73 @@ def get_frame_possible_urls(dataset_info, user_dir, output_dir, frame_type):
|
|
44
48
|
urls = {"user": None, "dataset": None, "output": None}
|
45
49
|
|
46
50
|
if user_dir is not None:
|
47
|
-
urls["user"] = make_dataurl(user_dir)
|
51
|
+
urls["user"] = {frame_type: make_dataurl(user_dir, frame_type) for frame_type in frame_types}
|
48
52
|
|
49
53
|
# tomoscan.esrf.scan.hdf5scan.REDUCED_{DARKS|FLATS}_DATAURLS.file_path() is a relative path
|
50
54
|
# Create a absolute path instead
|
51
|
-
urls["dataset"] =
|
55
|
+
urls["dataset"] = {
|
56
|
+
frame_type: make_dataurl(os.path.dirname(h5scan.master_file), frame_type) for frame_type in frame_types
|
57
|
+
}
|
52
58
|
|
53
59
|
if output_dir is not None:
|
54
|
-
urls["output"] = make_dataurl(output_dir)
|
60
|
+
urls["output"] = {frame_type: make_dataurl(output_dir, frame_type) for frame_type in frame_types}
|
55
61
|
|
56
62
|
return urls
|
57
63
|
|
58
64
|
|
65
|
+
def save_reduced_frames(dataset_info, reduced_frames_arrays, reduced_frames_urls):
|
66
|
+
reduce_func = {"flats": np.median, "darks": np.mean} # TODO configurable ?
|
67
|
+
|
68
|
+
# Get "where to write". tomoscan expects a DataUrl
|
69
|
+
darks_flats_dir_url = reduced_frames_urls.get("user", None)
|
70
|
+
if darks_flats_dir_url is not None:
|
71
|
+
output_url = darks_flats_dir_url
|
72
|
+
elif is_writeable(os.path.dirname(reduced_frames_urls["dataset"]["flats"].file_path())):
|
73
|
+
output_url = reduced_frames_urls["dataset"]
|
74
|
+
else:
|
75
|
+
output_url = reduced_frames_urls["output"]
|
76
|
+
|
77
|
+
# Get the "ReducedFrameInfos" data structure expected by tomoscan
|
78
|
+
def _get_additional_info(frame_type):
|
79
|
+
electric_current = dataset_info.dataset_scanner.electric_current
|
80
|
+
count_time = dataset_info.dataset_scanner.count_time
|
81
|
+
if electric_current is not None:
|
82
|
+
electric_current = {
|
83
|
+
sl.start: reduce_func[frame_type](electric_current[sl]) for sl in dataset_info.frames_slices(frame_type)
|
84
|
+
}
|
85
|
+
electric_current = [electric_current[k] for k in sorted(electric_current.keys())]
|
86
|
+
if count_time is not None:
|
87
|
+
count_time = {
|
88
|
+
sl.start: reduce_func[frame_type](count_time[sl]) for sl in dataset_info.frames_slices(frame_type)
|
89
|
+
}
|
90
|
+
count_time = [count_time[k] for k in sorted(count_time.keys())]
|
91
|
+
info = ReducedFramesInfos()
|
92
|
+
info.count_time = count_time
|
93
|
+
info.machine_electric_current = electric_current
|
94
|
+
return info
|
95
|
+
|
96
|
+
flats_info = _get_additional_info("flats")
|
97
|
+
darks_info = _get_additional_info("darks")
|
98
|
+
|
99
|
+
# Call tomoscan to save the reduced frames
|
100
|
+
dataset_info.dataset_scanner.save_reduced_darks(
|
101
|
+
reduced_frames_arrays["darks"],
|
102
|
+
output_urls=[output_url["darks"]],
|
103
|
+
darks_infos=darks_info,
|
104
|
+
metadata_output_urls=[get_metadata_url(output_url["darks"], "darks")],
|
105
|
+
overwrite=True,
|
106
|
+
)
|
107
|
+
dataset_info.dataset_scanner.save_reduced_flats(
|
108
|
+
reduced_frames_arrays["flats"],
|
109
|
+
output_urls=[output_url["flats"]],
|
110
|
+
flats_infos=flats_info,
|
111
|
+
metadata_output_urls=[get_metadata_url(output_url["flats"], "flats")],
|
112
|
+
overwrite=True,
|
113
|
+
)
|
114
|
+
dataset_info.logger.info("Saved reduced darks/flats to %s" % output_url["flats"].file_path())
|
115
|
+
return output_url, flats_info, darks_info
|
116
|
+
|
117
|
+
|
59
118
|
def get_metadata_url(url, frame_type):
|
60
119
|
"""
|
61
120
|
Return the url of the metadata stored alongside flats/darks
|
@@ -79,12 +138,15 @@ def tomoscan_load_reduced_frames(dataset_info, frame_type, url):
|
|
79
138
|
)
|
80
139
|
|
81
140
|
|
82
|
-
def
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
141
|
+
def data_url_exists(data_url):
|
142
|
+
"""
|
143
|
+
Return true iff the file exists and the data URL is valid (i.e data/group is actually in the file)
|
144
|
+
"""
|
145
|
+
if not (os.path.isfile(data_url.file_path())):
|
146
|
+
return False
|
147
|
+
with HDF5File(data_url.file_path(), "r") as f:
|
148
|
+
path_exists = f.get(data_url.data_path(), default=None) is not None
|
149
|
+
return path_exists
|
88
150
|
|
89
151
|
|
90
152
|
# pylint: disable=E1136
|
@@ -107,111 +169,58 @@ def update_dataset_info_flats_darks(dataset_info, flatfield_mode, output_dir=Non
|
|
107
169
|
"""
|
108
170
|
if flatfield_mode is False:
|
109
171
|
return
|
110
|
-
|
172
|
+
|
111
173
|
frames_types = ["darks", "flats"]
|
174
|
+
reduced_frames_urls = get_frame_possible_urls(dataset_info, darks_flats_dir, output_dir)
|
175
|
+
|
176
|
+
def _compute_and_save_reduced_frames():
|
177
|
+
try:
|
178
|
+
dataset_info.flats = dataset_info.get_reduced_flats()
|
179
|
+
dataset_info.darks = dataset_info.get_reduced_darks()
|
180
|
+
except FileNotFoundError:
|
181
|
+
msg = "Could not find any flats and/or darks"
|
182
|
+
raise FileNotFoundError(msg)
|
183
|
+
_, flats_info, darks_info = save_reduced_frames(
|
184
|
+
dataset_info, {"darks": dataset_info.darks, "flats": dataset_info.flats}, reduced_frames_urls
|
185
|
+
)
|
186
|
+
dataset_info.flats_srcurrent = flats_info.machine_electric_current
|
112
187
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
reduced_frames = dict.fromkeys(frames_types, None)
|
118
|
-
|
119
|
-
#
|
120
|
-
# Try to load frames
|
121
|
-
#
|
122
|
-
def load_reduced_frame(url, frame_type, frames_loaded, reduced_frames):
|
123
|
-
if frames_loaded[frame_type]:
|
124
|
-
return
|
125
|
-
frames, info = tomoscan_load_reduced_frames(dataset_info, frame_type, url)
|
126
|
-
if frames not in (None, {}):
|
127
|
-
dataset_info.logger.info("Loaded %s from %s" % (frame_type, url.file_path()))
|
128
|
-
frames_loaded[frame_type] = True
|
129
|
-
reduced_frames[frame_type] = frames, info
|
130
|
-
else:
|
131
|
-
msg = "Could not load %s from %s" % (frame_type, url.file_path())
|
132
|
-
logger.error(msg)
|
133
|
-
|
134
|
-
frames_loaded = dict.fromkeys(frames_types, False)
|
135
|
-
if flatfield_mode != "force-compute":
|
136
|
-
for load_from in ["user", "dataset", "output"]: # in that order
|
137
|
-
for frame_type in frames_types:
|
138
|
-
url = reduced_frames_urls[frame_type][load_from]
|
139
|
-
if url is None:
|
140
|
-
continue # cannot load from this source (eg. undefined folder)
|
141
|
-
load_reduced_frame(url, frame_type, frames_loaded, reduced_frames)
|
142
|
-
if all(frames_loaded.values()):
|
143
|
-
break
|
144
|
-
|
145
|
-
if not all(frames_loaded.values()) and flatfield_mode == "force-load":
|
146
|
-
raise ValueError("Could not load darks/flats (using 'force-load')")
|
188
|
+
if flatfield_mode == "force-compute":
|
189
|
+
_compute_and_save_reduced_frames()
|
190
|
+
return
|
147
191
|
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
if reduced_frames["darks"][0] == {} or reduced_frames["flats"][0] == {}:
|
162
|
-
raise ValueError(
|
163
|
-
"Could not get any reduced flat/dark. This probably means that no already-reduced flats/darks were found and that the dataset itself does not have any flat/dark"
|
164
|
-
)
|
192
|
+
def _can_load_from(folder_type):
|
193
|
+
if reduced_frames_urls.get(folder_type, None) is None:
|
194
|
+
return False
|
195
|
+
return all([data_url_exists(reduced_frames_urls[folder_type][frame_type]) for frame_type in frames_types])
|
196
|
+
|
197
|
+
where_to_load_from = None
|
198
|
+
if reduced_frames_urls["user"] is not None and _can_load_from("user"):
|
199
|
+
where_to_load_from = "user"
|
200
|
+
elif _can_load_from("dataset"):
|
201
|
+
where_to_load_from = "dataset"
|
202
|
+
elif _can_load_from("output"):
|
203
|
+
where_to_load_from = "output"
|
165
204
|
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
# COMPAT.
|
189
|
-
if frames_urls[frame_type] is None:
|
190
|
-
frames_urls[frame_type] = tomoscan_load_reduced_frames(dataset_info, frame_type, url)
|
191
|
-
#
|
192
|
-
if all(frames_saved.values()):
|
193
|
-
break
|
194
|
-
|
195
|
-
dataset_info.flats = frames_urls["flats"][0] # reduced_frames["flats"] # in future versions
|
196
|
-
dataset_info.flats_srcurrent = frames_urls["flats"][1].machine_electric_current
|
197
|
-
# This is an extra check to avoid having more than 1 (reduced) dark.
|
198
|
-
# FlatField only works with exactly 1 (reduced) dark (having more than 1 series of darks makes little sense)
|
199
|
-
# This is normally prevented by tomoscan HDF5FramesReducer, but let's add this extra check
|
200
|
-
darks_ = frames_urls["darks"][0] # reduced_frames["darks"] # in future versions
|
201
|
-
if len(darks_) > 1:
|
202
|
-
dark_idx = sorted(darks_.keys())[0]
|
203
|
-
dataset_info.logger.error("Found more that one series of darks. Keeping only the first one")
|
204
|
-
darks_ = {dark_idx: darks_[dark_idx]}
|
205
|
-
#
|
206
|
-
dataset_info.darks = darks_
|
207
|
-
dataset_info.darks_srcurrent = frames_urls["darks"][1].machine_electric_current
|
208
|
-
|
209
|
-
|
210
|
-
# tomoscan "compute_reduced_XX" is quite slow. If needed, here is an alternative implementation
|
211
|
-
def my_reduce_flats(di):
|
212
|
-
res = {}
|
213
|
-
with HDF5File(di.dataset_hdf5_url.file_path(), "r") as f:
|
214
|
-
for data_slice in di.get_data_slices("flats"):
|
215
|
-
data = f[di.dataset_hdf5_url.data_path()][data_slice.start : data_slice.stop]
|
216
|
-
res[data_slice.start] = np.median(data, axis=0)
|
217
|
-
return res
|
205
|
+
if where_to_load_from == None and flatfield_mode == "force-load":
|
206
|
+
raise ValueError("Could not load darks/flats (using 'force-load')")
|
207
|
+
|
208
|
+
if where_to_load_from is not None:
|
209
|
+
reduced_frames_with_info = {}
|
210
|
+
for frame_type in frames_types:
|
211
|
+
reduced_frames_with_info[frame_type] = tomoscan_load_reduced_frames(
|
212
|
+
dataset_info, frame_type, reduced_frames_urls[where_to_load_from][frame_type]
|
213
|
+
)
|
214
|
+
dataset_info.logger.info(
|
215
|
+
"Loaded %s from %s" % (frame_type, reduced_frames_urls[where_to_load_from][frame_type].file_path())
|
216
|
+
)
|
217
|
+
red_frames_dict, red_frames_info = reduced_frames_with_info[frame_type]
|
218
|
+
setattr(
|
219
|
+
dataset_info,
|
220
|
+
frame_type,
|
221
|
+
{k: get_data(red_frames_dict[k]) for k in red_frames_dict.keys()},
|
222
|
+
)
|
223
|
+
if frame_type == "flats":
|
224
|
+
dataset_info.flats_srcurrent = red_frames_info.machine_electric_current
|
225
|
+
else:
|
226
|
+
_compute_and_save_reduced_frames()
|