nabu 2024.1.9__py3-none-any.whl → 2024.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nabu/__init__.py +1 -1
- nabu/app/bootstrap.py +2 -3
- nabu/app/cast_volume.py +4 -2
- nabu/app/cli_configs.py +5 -0
- nabu/app/composite_cor.py +1 -1
- nabu/app/create_distortion_map_from_poly.py +5 -6
- nabu/app/diag_to_pix.py +7 -19
- nabu/app/diag_to_rot.py +14 -29
- nabu/app/double_flatfield.py +32 -44
- nabu/app/parse_reconstruction_log.py +3 -0
- nabu/app/reconstruct.py +53 -15
- nabu/app/reconstruct_helical.py +2 -2
- nabu/app/stitching.py +27 -13
- nabu/app/tests/test_reduce_dark_flat.py +4 -1
- nabu/cuda/kernel.py +11 -2
- nabu/cuda/processing.py +2 -2
- nabu/cuda/src/cone.cu +77 -0
- nabu/cuda/src/hierarchical_backproj.cu +271 -0
- nabu/cuda/utils.py +0 -6
- nabu/estimation/alignment.py +5 -19
- nabu/estimation/cor.py +173 -599
- nabu/estimation/cor_sino.py +356 -26
- nabu/estimation/focus.py +63 -11
- nabu/estimation/tests/test_cor.py +124 -58
- nabu/estimation/tests/test_focus.py +6 -6
- nabu/estimation/tilt.py +2 -1
- nabu/estimation/utils.py +5 -33
- nabu/io/__init__.py +1 -1
- nabu/io/cast_volume.py +1 -1
- nabu/io/reader.py +416 -21
- nabu/io/tests/test_readers.py +422 -0
- nabu/io/tests/test_writers.py +1 -102
- nabu/io/writer.py +4 -433
- nabu/opencl/kernel.py +14 -3
- nabu/opencl/processing.py +8 -0
- nabu/pipeline/config_validators.py +5 -2
- nabu/pipeline/datadump.py +12 -5
- nabu/pipeline/estimators.py +162 -188
- nabu/pipeline/fullfield/chunked.py +168 -92
- nabu/pipeline/fullfield/chunked_cuda.py +7 -3
- nabu/pipeline/fullfield/computations.py +2 -7
- nabu/pipeline/fullfield/dataset_validator.py +0 -4
- nabu/pipeline/fullfield/nabu_config.py +37 -13
- nabu/pipeline/fullfield/processconfig.py +22 -13
- nabu/pipeline/fullfield/reconstruction.py +13 -9
- nabu/pipeline/helical/helical_chunked_regridded.py +1 -1
- nabu/pipeline/helical/helical_chunked_regridded_cuda.py +1 -0
- nabu/pipeline/helical/helical_reconstruction.py +1 -1
- nabu/pipeline/params.py +21 -1
- nabu/pipeline/processconfig.py +1 -12
- nabu/pipeline/reader.py +146 -0
- nabu/pipeline/tests/test_estimators.py +44 -72
- nabu/pipeline/utils.py +4 -2
- nabu/pipeline/writer.py +10 -2
- nabu/preproc/ccd_cuda.py +1 -1
- nabu/preproc/ctf.py +14 -7
- nabu/preproc/ctf_cuda.py +2 -3
- nabu/preproc/double_flatfield.py +5 -12
- nabu/preproc/double_flatfield_cuda.py +2 -2
- nabu/preproc/flatfield.py +5 -1
- nabu/preproc/flatfield_cuda.py +5 -1
- nabu/preproc/phase.py +24 -73
- nabu/preproc/phase_cuda.py +5 -8
- nabu/preproc/tests/test_ctf.py +11 -7
- nabu/preproc/tests/test_flatfield.py +67 -122
- nabu/preproc/tests/test_paganin.py +54 -30
- nabu/processing/azim.py +206 -0
- nabu/processing/convolution_cuda.py +1 -1
- nabu/processing/fft_cuda.py +15 -17
- nabu/processing/histogram.py +2 -0
- nabu/processing/histogram_cuda.py +2 -1
- nabu/processing/kernel_base.py +3 -0
- nabu/processing/muladd_cuda.py +1 -0
- nabu/processing/padding_opencl.py +1 -1
- nabu/processing/roll_opencl.py +1 -0
- nabu/processing/rotation_cuda.py +2 -2
- nabu/processing/tests/test_fft.py +17 -10
- nabu/processing/unsharp_cuda.py +1 -1
- nabu/reconstruction/cone.py +104 -40
- nabu/reconstruction/fbp.py +3 -0
- nabu/reconstruction/fbp_base.py +7 -2
- nabu/reconstruction/filtering.py +20 -7
- nabu/reconstruction/filtering_cuda.py +7 -1
- nabu/reconstruction/hbp.py +424 -0
- nabu/reconstruction/mlem.py +99 -0
- nabu/reconstruction/reconstructor.py +2 -0
- nabu/reconstruction/rings_cuda.py +19 -19
- nabu/reconstruction/sinogram_cuda.py +1 -0
- nabu/reconstruction/sinogram_opencl.py +3 -1
- nabu/reconstruction/tests/test_cone.py +10 -5
- nabu/reconstruction/tests/test_deringer.py +7 -6
- nabu/reconstruction/tests/test_fbp.py +124 -10
- nabu/reconstruction/tests/test_filtering.py +13 -11
- nabu/reconstruction/tests/test_halftomo.py +30 -4
- nabu/reconstruction/tests/test_mlem.py +91 -0
- nabu/reconstruction/tests/test_reconstructor.py +8 -3
- nabu/resources/dataset_analyzer.py +142 -92
- nabu/resources/gpu.py +1 -0
- nabu/resources/nxflatfield.py +134 -125
- nabu/resources/templates/id16a_fluo.conf +42 -0
- nabu/resources/tests/test_extract.py +10 -0
- nabu/resources/tests/test_nxflatfield.py +2 -2
- nabu/stitching/alignment.py +80 -24
- nabu/stitching/config.py +105 -68
- nabu/stitching/definitions.py +1 -0
- nabu/stitching/frame_composition.py +68 -60
- nabu/stitching/overlap.py +91 -51
- nabu/stitching/single_axis_stitching.py +32 -0
- nabu/stitching/slurm_utils.py +6 -6
- nabu/stitching/stitcher/__init__.py +0 -0
- nabu/stitching/stitcher/base.py +124 -0
- nabu/stitching/stitcher/dumper/__init__.py +3 -0
- nabu/stitching/stitcher/dumper/base.py +94 -0
- nabu/stitching/stitcher/dumper/postprocessing.py +356 -0
- nabu/stitching/stitcher/dumper/preprocessing.py +60 -0
- nabu/stitching/stitcher/post_processing.py +555 -0
- nabu/stitching/stitcher/pre_processing.py +1068 -0
- nabu/stitching/stitcher/single_axis.py +484 -0
- nabu/stitching/stitcher/stitcher.py +0 -0
- nabu/stitching/stitcher/y_stitcher.py +13 -0
- nabu/stitching/stitcher/z_stitcher.py +45 -0
- nabu/stitching/stitcher_2D.py +278 -0
- nabu/stitching/tests/test_config.py +12 -37
- nabu/stitching/tests/test_frame_composition.py +33 -59
- nabu/stitching/tests/test_overlap.py +149 -7
- nabu/stitching/tests/test_utils.py +1 -1
- nabu/stitching/tests/test_y_preprocessing_stitching.py +132 -0
- nabu/stitching/tests/{test_z_stitching.py → test_z_postprocessing_stitching.py} +167 -561
- nabu/stitching/tests/test_z_preprocessing_stitching.py +431 -0
- nabu/stitching/utils/__init__.py +1 -0
- nabu/stitching/utils/post_processing.py +281 -0
- nabu/stitching/utils/tests/test_post-processing.py +21 -0
- nabu/stitching/{utils.py → utils/utils.py} +79 -52
- nabu/stitching/y_stitching.py +27 -0
- nabu/stitching/z_stitching.py +32 -2263
- nabu/testutils.py +1 -152
- nabu/thirdparty/tomocupy_remove_stripe.py +43 -9
- nabu/utils.py +158 -61
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/METADATA +10 -3
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/RECORD +144 -121
- nabu/io/tiffwriter_zmm.py +0 -99
- nabu/pipeline/fallback_utils.py +0 -149
- nabu/pipeline/helical/tests/test_accumulator.py +0 -158
- nabu/pipeline/helical/tests/test_pipeline_elements_full.py +0 -355
- nabu/pipeline/helical/tests/test_strategy.py +0 -61
- nabu/pipeline/helical/utils.py +0 -51
- nabu/pipeline/tests/test_chunk_reader.py +0 -74
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/LICENSE +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/WHEEL +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/entry_points.txt +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/top_level.txt +0 -0
@@ -17,7 +17,6 @@ from nxtomo.nxobject.nxdetector import ImageKey
|
|
17
17
|
|
18
18
|
|
19
19
|
class ProcessConfig(ProcessConfigBase):
|
20
|
-
|
21
20
|
"""
|
22
21
|
A ProcessConfig object has these main fields:
|
23
22
|
- dataset_info: information about the current dataset
|
@@ -57,7 +56,7 @@ class ProcessConfig(ProcessConfigBase):
|
|
57
56
|
"flatfield",
|
58
57
|
"ccd_correction",
|
59
58
|
"double_flatfield",
|
60
|
-
"
|
59
|
+
"tilt_correction",
|
61
60
|
"phase",
|
62
61
|
"unsharp_mask",
|
63
62
|
"take_log",
|
@@ -74,13 +73,16 @@ class ProcessConfig(ProcessConfigBase):
|
|
74
73
|
Update the 'dataset_info' (DatasetAnalyzer class instance) data structure with options from user configuration.
|
75
74
|
"""
|
76
75
|
self.logger.debug("Updating dataset information with user configuration")
|
77
|
-
if self.dataset_info.kind == "
|
76
|
+
if self.dataset_info.kind == "nx":
|
78
77
|
update_dataset_info_flats_darks(
|
79
78
|
self.dataset_info,
|
80
79
|
self.nabu_config["preproc"]["flatfield"],
|
81
80
|
output_dir=self.nabu_config["output"]["location"],
|
82
81
|
darks_flats_dir=self.nabu_config["dataset"]["darks_flats_dir"],
|
83
82
|
)
|
83
|
+
elif self.dataset_info.kind == "edf":
|
84
|
+
self.dataset_info.flats = self.dataset_info.get_reduced_flats()
|
85
|
+
self.dataset_info.darks = self.dataset_info.get_reduced_darks()
|
84
86
|
self.rec_params = self.nabu_config["reconstruction"]
|
85
87
|
|
86
88
|
subsampling_factor, subsampling_start = self.nabu_config["dataset"]["projections_subsampling"]
|
@@ -361,7 +363,6 @@ class ProcessConfig(ProcessConfigBase):
|
|
361
363
|
method: str or None
|
362
364
|
Rotation method: one of the values of `nabu.resources.params.radios_rotation_mode`
|
363
365
|
"""
|
364
|
-
user_rotate_projections = self.nabu_config["preproc"]["rotate_projections"]
|
365
366
|
tilt = self.dataset_info.detector_tilt
|
366
367
|
phase_method = self.nabu_config["phase"]["method"]
|
367
368
|
do_ctf = phase_method == "CTF"
|
@@ -369,7 +370,7 @@ class ProcessConfig(ProcessConfigBase):
|
|
369
370
|
do_unsharp = (
|
370
371
|
self.nabu_config["phase"]["unsharp_method"] is not None and self.nabu_config["phase"]["unsharp_coeff"] > 0
|
371
372
|
)
|
372
|
-
if
|
373
|
+
if tilt is None:
|
373
374
|
return None
|
374
375
|
if do_ctf:
|
375
376
|
return "full"
|
@@ -397,7 +398,6 @@ class ProcessConfig(ProcessConfigBase):
|
|
397
398
|
# For now data is assumed to be on disk (see issue #66).
|
398
399
|
tasks.append("read_chunk")
|
399
400
|
options["read_chunk"] = {
|
400
|
-
"files": dataset_info.projections,
|
401
401
|
"sub_region": None,
|
402
402
|
"binning": binning,
|
403
403
|
"dataset_subsampling": nabu_config["dataset"]["projections_subsampling"],
|
@@ -408,7 +408,7 @@ class ProcessConfig(ProcessConfigBase):
|
|
408
408
|
if nabu_config["preproc"]["flatfield"]:
|
409
409
|
tasks.append("flatfield")
|
410
410
|
options["flatfield"] = {
|
411
|
-
#
|
411
|
+
# Data reader handles binning/subsampling by itself,
|
412
412
|
# but FlatField needs "real" indices (after binning/subsampling)
|
413
413
|
"projs_indices": self.projs_indices(subsampling=False),
|
414
414
|
"binning": binning,
|
@@ -436,6 +436,9 @@ class ProcessConfig(ProcessConfigBase):
|
|
436
436
|
"flats_srcurrent": flats_srcurrent,
|
437
437
|
}
|
438
438
|
)
|
439
|
+
if len(dataset_info.darks) > 1:
|
440
|
+
self.logger.warning("Cannot do flat-field with more than one reduced dark. Taking the first one.")
|
441
|
+
dataset_info.darks = dataset_info.darks[sorted(dataset_info.darks.keys())[0]]
|
439
442
|
#
|
440
443
|
# Spikes filter
|
441
444
|
#
|
@@ -460,9 +463,9 @@ class ProcessConfig(ProcessConfigBase):
|
|
460
463
|
# Radios rotation (do it here if possible)
|
461
464
|
#
|
462
465
|
if self.get_radios_rotation_mode() == "chunk":
|
463
|
-
tasks.append("
|
464
|
-
options["
|
465
|
-
"angle": nabu_config["preproc"]["
|
466
|
+
tasks.append("tilt_correction")
|
467
|
+
options["tilt_correction"] = {
|
468
|
+
"angle": nabu_config["preproc"]["tilt_correction"] or dataset_info.detector_tilt,
|
466
469
|
"center": nabu_config["preproc"]["rotate_projections_center"],
|
467
470
|
"mode": "chunk",
|
468
471
|
}
|
@@ -508,9 +511,9 @@ class ProcessConfig(ProcessConfigBase):
|
|
508
511
|
# Radios rotation (do it here if mode=="full")
|
509
512
|
#
|
510
513
|
if self.get_radios_rotation_mode() == "full":
|
511
|
-
tasks.append("
|
512
|
-
options["
|
513
|
-
"angle": nabu_config["preproc"]["
|
514
|
+
tasks.append("tilt_correction")
|
515
|
+
options["tilt_correction"] = {
|
516
|
+
"angle": nabu_config["preproc"]["tilt_correction"] or dataset_info.detector_tilt,
|
514
517
|
"center": nabu_config["preproc"]["rotate_projections_center"],
|
515
518
|
"mode": "full",
|
516
519
|
}
|
@@ -555,6 +558,7 @@ class ProcessConfig(ProcessConfigBase):
|
|
555
558
|
self.rec_params,
|
556
559
|
[
|
557
560
|
"method",
|
561
|
+
"implementation",
|
558
562
|
"fbp_filter_type",
|
559
563
|
"fbp_filter_cutoff",
|
560
564
|
"padding_type",
|
@@ -566,8 +570,11 @@ class ProcessConfig(ProcessConfigBase):
|
|
566
570
|
"end_z",
|
567
571
|
"centered_axis",
|
568
572
|
"clip_outer_circle",
|
573
|
+
"outer_circle_value",
|
569
574
|
"source_sample_dist",
|
570
575
|
"sample_detector_dist",
|
576
|
+
"hbp_legs",
|
577
|
+
"hbp_reduction_steps",
|
571
578
|
],
|
572
579
|
)
|
573
580
|
rec_options = options["reconstruction"]
|
@@ -586,6 +593,8 @@ class ProcessConfig(ProcessConfigBase):
|
|
586
593
|
voxel_size,
|
587
594
|
) # pix size is in microns in dataset_info
|
588
595
|
|
596
|
+
rec_options["iterations"] = nabu_config["reconstruction"]["iterations"]
|
597
|
+
|
589
598
|
# x/y/z position information
|
590
599
|
def get_mean_pos(position_array):
|
591
600
|
if position_array is None:
|
@@ -146,7 +146,9 @@ class FullFieldReconstructor:
|
|
146
146
|
else:
|
147
147
|
self.gpu_mem = self.resources["gpus"][self._gpu_id]["memory_GB"] * self.gpu_mem_fraction
|
148
148
|
if backend == "cuda":
|
149
|
-
|
149
|
+
if not (__has_pycuda__):
|
150
|
+
raise RuntimeError("pycuda not avilable")
|
151
|
+
self._pipeline_cls = CudaChunkedPipeline # pylint: disable=E0606
|
150
152
|
self.backend = backend
|
151
153
|
|
152
154
|
def _compute_max_chunk_size(self):
|
@@ -251,7 +253,7 @@ class FullFieldReconstructor:
|
|
251
253
|
if self.process_config.processing_options.get("phase", {}).get("method", None) == "CTF":
|
252
254
|
force_grouped_mode = True
|
253
255
|
msg = "CTF phase retrieval needs to process full radios"
|
254
|
-
if self.process_config.processing_options.get("
|
256
|
+
if self.process_config.processing_options.get("tilt_correction", {}).get("angle", 0) > 15:
|
255
257
|
force_grouped_mode = True
|
256
258
|
msg = "Radios rotation with a large angle needs to process full radios"
|
257
259
|
if self.process_config.resume_from_step == "sinogram" and force_grouped_mode:
|
@@ -263,7 +265,7 @@ class FullFieldReconstructor:
|
|
263
265
|
self.chunk_shape = (self.n_angles, self.chunk_size, self.n_x)
|
264
266
|
else:
|
265
267
|
# Fall-back mode (slower)
|
266
|
-
self.logger.warning(msg)
|
268
|
+
self.logger.warning(msg) # pylint: disable=E0606
|
267
269
|
self._pipeline_mode = "grouped"
|
268
270
|
self._compute_max_group_size()
|
269
271
|
self.chunk_shape = (self.group_size, self.delta_z, self.n_x)
|
@@ -286,9 +288,10 @@ class FullFieldReconstructor:
|
|
286
288
|
phase_margin = self._compute_phase_margin()
|
287
289
|
translations_margin = self._compute_translations_margin()
|
288
290
|
cone_margin = self._compute_cone_overlap()
|
291
|
+
rot_margin = self._compute_rotation_margin()
|
289
292
|
# TODO radios rotation/movements
|
290
|
-
margin_v = max(unsharp_margin[0], phase_margin[0], translations_margin[0], cone_margin[0])
|
291
|
-
margin_h = max(unsharp_margin[1], phase_margin[1], translations_margin[1], cone_margin[1])
|
293
|
+
margin_v = max(unsharp_margin[0], phase_margin[0], translations_margin[0], cone_margin[0], rot_margin[0])
|
294
|
+
margin_h = max(unsharp_margin[1], phase_margin[1], translations_margin[1], cone_margin[1], rot_margin[1])
|
292
295
|
if margin_v > 0:
|
293
296
|
self.logger.info("Estimated margin: %d pixels" % margin_v)
|
294
297
|
|
@@ -360,7 +363,7 @@ class FullFieldReconstructor:
|
|
360
363
|
return (max_overlap, 0)
|
361
364
|
|
362
365
|
def _compute_rotation_margin(self):
|
363
|
-
if "
|
366
|
+
if "tilt_correction" in self.process_config.processing_steps:
|
364
367
|
# Partial radios rotation yields too much error in single-slice mode
|
365
368
|
# Forcing a big margin circumvents the problem
|
366
369
|
# This is likely to trigger the 'grouped mode', but perhaps grouped mode should always be used when rotating radios
|
@@ -505,8 +508,8 @@ class FullFieldReconstructor:
|
|
505
508
|
|
506
509
|
def _print_tasks(self):
|
507
510
|
for task in self.tasks:
|
508
|
-
margin_up, margin_down = task["
|
509
|
-
s_u, s_d = task["sub_region"]
|
511
|
+
margin_up, margin_down = task["margin"][0]
|
512
|
+
s_u, s_d = task["sub_region"][1]
|
510
513
|
print(
|
511
514
|
"Top Margin: [%04d, %04d[ | Slices: [%04d, %04d[ | Bottom Margin: [%04d, %04d[" # pylint: disable=E1307
|
512
515
|
% (s_u, s_u + margin_up, s_u + margin_up, s_d - margin_down, s_d - margin_down, s_d)
|
@@ -713,12 +716,13 @@ class FullFieldReconstructor:
|
|
713
716
|
# (these values are checked in ProcessConfig._configure_resume())
|
714
717
|
#
|
715
718
|
patched_start_end_z = False
|
716
|
-
user_rec_config = self.process_config.processing_options["reconstruction"]
|
717
719
|
if (
|
718
720
|
self._margin_v > 0
|
719
721
|
and process_name != "reconstruction"
|
720
722
|
and self.process_config.is_before_radios_cropping(process_name)
|
723
|
+
and "reconstruction" in self.process_config.processing_steps
|
721
724
|
):
|
725
|
+
user_rec_config = self.process_config.processing_options["reconstruction"]
|
722
726
|
patched_start_end_z = True
|
723
727
|
old_start_z = user_rec_config["start_z"]
|
724
728
|
old_end_z = user_rec_config["end_z"]
|
@@ -1008,7 +1008,7 @@ class HelicalChunkedRegriddedPipeline:
|
|
1008
1008
|
np.array((subr_start_z, subr_end_z), "i"),
|
1009
1009
|
np.array((dtasrc_start_z, dtasrc_end_z), "i"),
|
1010
1010
|
data_raw,
|
1011
|
-
radios_angular_range_slicing # my_subsampled_indexes is important in order to compare the
|
1011
|
+
radios_angular_range_slicing, # my_subsampled_indexes is important in order to compare the
|
1012
1012
|
# radios positions with respect to the flat position, and these position
|
1013
1013
|
# are given as the sequential acquisition number which counts everything ( flats, darks, radios )
|
1014
1014
|
# Insteqd, in order to access array which spans only the radios, we need to have an idea of where we are.
|
@@ -468,7 +468,7 @@ class HelicalReconstructorRegridded:
|
|
468
468
|
phase_margin=task["phase_margin"],
|
469
469
|
reading_granularity=self.reading_granularity,
|
470
470
|
span_info=self._span_info,
|
471
|
-
diag_zpro_run=self.diag_zpro_run
|
471
|
+
diag_zpro_run=self.diag_zpro_run,
|
472
472
|
# cuda_options=self.cuda_options
|
473
473
|
)
|
474
474
|
|
nabu/pipeline/params.py
CHANGED
@@ -33,7 +33,18 @@ padding_modes = {
|
|
33
33
|
"zero": "zeros",
|
34
34
|
}
|
35
35
|
|
36
|
-
reconstruction_methods = {
|
36
|
+
reconstruction_methods = {
|
37
|
+
"fbp": "FBP",
|
38
|
+
"cone": "cone",
|
39
|
+
"conic": "cone",
|
40
|
+
"none": None,
|
41
|
+
"": None,
|
42
|
+
"mlem": "mlem",
|
43
|
+
"fluo": "mlem",
|
44
|
+
"em": "mlem",
|
45
|
+
"hbp": "HBP",
|
46
|
+
"ghbp": "HBP",
|
47
|
+
}
|
37
48
|
|
38
49
|
fbp_filters = {
|
39
50
|
"ramlak": "ramlak",
|
@@ -64,6 +75,14 @@ optim_algorithms = {
|
|
64
75
|
"fista": "fista",
|
65
76
|
}
|
66
77
|
|
78
|
+
reco_implementations = {
|
79
|
+
"astra": "astra",
|
80
|
+
"corrct": "corrct",
|
81
|
+
"corr-ct": "corrct",
|
82
|
+
"nabu": "nabu",
|
83
|
+
"": None,
|
84
|
+
}
|
85
|
+
|
67
86
|
files_formats = {
|
68
87
|
"h5": "hdf5",
|
69
88
|
"hdf5": "hdf5",
|
@@ -123,6 +142,7 @@ cor_methods = {
|
|
123
142
|
"fourier-angle": "fourier-angles",
|
124
143
|
"fourier angle": "fourier-angles",
|
125
144
|
"octave-accurate": "octave-accurate",
|
145
|
+
"vo": "vo",
|
126
146
|
}
|
127
147
|
|
128
148
|
|
nabu/pipeline/processconfig.py
CHANGED
@@ -168,18 +168,7 @@ class ProcessConfigBase:
|
|
168
168
|
|
169
169
|
def _get_tilt(self):
|
170
170
|
tilt = self.nabu_config["preproc"]["tilt_correction"]
|
171
|
-
|
172
|
-
if user_rot_projs is not None and tilt is not None:
|
173
|
-
msg = "=" * 80 + "\n"
|
174
|
-
msg += (
|
175
|
-
"Both 'detector_tilt' and 'rotate_projections' options were provided. The option 'rotate_projections' will take precedence. This means that the projections will be rotated by %f degrees and the option 'detector_tilt' will be ignored."
|
176
|
-
% user_rot_projs
|
177
|
-
)
|
178
|
-
msg += "\n" + "=" * 80
|
179
|
-
self.logger.warning(msg)
|
180
|
-
tilt = user_rot_projs
|
181
|
-
#
|
182
|
-
if isinstance(tilt, str): # auto-tilt
|
171
|
+
if isinstance(tilt, str): # auto-tilt...
|
183
172
|
self.tilt_estimator = DetectorTiltEstimator(
|
184
173
|
self.dataset_info,
|
185
174
|
do_flatfield=self.nabu_config["preproc"]["flatfield"],
|
nabu/pipeline/reader.py
ADDED
@@ -0,0 +1,146 @@
|
|
1
|
+
from multiprocessing.pool import ThreadPool
|
2
|
+
import numpy as np
|
3
|
+
|
4
|
+
from nabu.utils import get_num_threads
|
5
|
+
from ..misc.binning import binning as image_binning
|
6
|
+
from ..io.reader import NXTomoReader, EDFStackReader
|
7
|
+
|
8
|
+
|
9
|
+
#
|
10
|
+
# NXTomoReader with binning
|
11
|
+
#
|
12
|
+
def bin_image_stack(src_stack, dst_stack, binning_factor=(2, 2), num_threads=8):
|
13
|
+
def _apply_binning(img_res_tuple):
|
14
|
+
img, res = img_res_tuple
|
15
|
+
res[:] = image_binning(img, binning_factor)
|
16
|
+
|
17
|
+
if dst_stack is None:
|
18
|
+
dst_stack = np.zeros((src_stack.shape[0],) + image_binning(src_stack[0], binning_factor).shape, dtype="f")
|
19
|
+
|
20
|
+
with ThreadPool(num_threads) as tp:
|
21
|
+
tp.map(_apply_binning, zip(src_stack, dst_stack))
|
22
|
+
return dst_stack
|
23
|
+
|
24
|
+
|
25
|
+
def NXTomoReaderBinning(binning_factor, *nxtomoreader_args, num_threads=None, **nxtomoreader_kwargs):
|
26
|
+
[
|
27
|
+
nxtomoreader_kwargs.pop(kwarg, None)
|
28
|
+
for kwarg in ["processing_func", "processing_func_args", "processing_func_kwargs"]
|
29
|
+
]
|
30
|
+
nxtomoreader_kwargs["processing_func"] = bin_image_stack
|
31
|
+
nxtomoreader_kwargs["processing_func_kwargs"] = {
|
32
|
+
"binning_factor": binning_factor,
|
33
|
+
"num_threads": num_threads or get_num_threads(),
|
34
|
+
}
|
35
|
+
|
36
|
+
return NXTomoReader(
|
37
|
+
*nxtomoreader_args,
|
38
|
+
**nxtomoreader_kwargs,
|
39
|
+
)
|
40
|
+
|
41
|
+
|
42
|
+
#
|
43
|
+
# NXTomoReader with distortion correction
|
44
|
+
#
|
45
|
+
|
46
|
+
|
47
|
+
def apply_distortion_correction_on_images_stack(src_stack, dst_stack, distortion_corrector, num_threads=8):
|
48
|
+
_, subregion = distortion_corrector.get_actual_shapes_source_target()
|
49
|
+
src_x_start, src_x_end, src_z_start, src_z_end = subregion
|
50
|
+
if dst_stack is None:
|
51
|
+
dst_stack = np.zeros([src_stack.shape[0], src_z_end - src_z_start, src_x_end - src_x_start], "f")
|
52
|
+
|
53
|
+
def apply_corrector(i_img_tuple):
|
54
|
+
i, img = i_img_tuple
|
55
|
+
dst_stack[i] = distortion_corrector.transform(img)
|
56
|
+
|
57
|
+
with ThreadPool(num_threads) as tp:
|
58
|
+
tp.map(apply_corrector, enumerate(src_stack))
|
59
|
+
|
60
|
+
return dst_stack
|
61
|
+
|
62
|
+
|
63
|
+
def NXTomoReaderDistortionCorrection(distortion_corrector, *nxtomoreader_args, num_threads=None, **nxtomoreader_kwargs):
|
64
|
+
[
|
65
|
+
nxtomoreader_kwargs.pop(kwarg, None)
|
66
|
+
for kwarg in ["processing_func", "processing_func_args", "processing_func_kwargs"]
|
67
|
+
]
|
68
|
+
nxtomoreader_kwargs["processing_func"] = apply_distortion_correction_on_images_stack
|
69
|
+
nxtomoreader_kwargs["processing_func_args"] = [distortion_corrector]
|
70
|
+
nxtomoreader_kwargs["processing_func_kwargs"] = {"num_threads": num_threads or get_num_threads()}
|
71
|
+
|
72
|
+
return NXTomoReader(
|
73
|
+
*nxtomoreader_args,
|
74
|
+
**nxtomoreader_kwargs,
|
75
|
+
)
|
76
|
+
|
77
|
+
|
78
|
+
#
|
79
|
+
# EDF Reader with binning
|
80
|
+
#
|
81
|
+
|
82
|
+
|
83
|
+
def EDFStackReaderBinning(binning_factor, *edfstackreader_args, **edfstackreader_kwargs):
|
84
|
+
[
|
85
|
+
edfstackreader_kwargs.pop(kwarg, None)
|
86
|
+
for kwarg in ["processing_func", "processing_func_args", "processing_func_kwargs"]
|
87
|
+
]
|
88
|
+
edfstackreader_kwargs["processing_func"] = image_binning
|
89
|
+
edfstackreader_kwargs["processing_func_args"] = [binning_factor]
|
90
|
+
|
91
|
+
return EDFStackReader(
|
92
|
+
*edfstackreader_args,
|
93
|
+
**edfstackreader_kwargs,
|
94
|
+
)
|
95
|
+
|
96
|
+
|
97
|
+
#
|
98
|
+
# EDF Reader with distortion correction
|
99
|
+
#
|
100
|
+
|
101
|
+
|
102
|
+
def apply_distortion_correction_on_image(image, distortion_corrector):
|
103
|
+
return distortion_corrector.transform(image)
|
104
|
+
|
105
|
+
|
106
|
+
def EDFStackReaderDistortionCorrection(distortion_corrector, *edfstackreader_args, **edfstackreader_kwargs):
|
107
|
+
[
|
108
|
+
edfstackreader_kwargs.pop(kwarg, None)
|
109
|
+
for kwarg in ["processing_func", "processing_func_args", "processing_func_kwargs"]
|
110
|
+
]
|
111
|
+
edfstackreader_kwargs["processing_func"] = apply_distortion_correction_on_image
|
112
|
+
edfstackreader_kwargs["processing_func_args"] = [distortion_corrector]
|
113
|
+
|
114
|
+
return EDFStackReader(
|
115
|
+
*edfstackreader_args,
|
116
|
+
**edfstackreader_kwargs,
|
117
|
+
)
|
118
|
+
|
119
|
+
|
120
|
+
def load_darks_flats(
|
121
|
+
dataset_info, sub_region, processing_func=None, processing_func_args=None, processing_func_kwargs=None
|
122
|
+
):
|
123
|
+
"""
|
124
|
+
Load the (reduced) darks and flats and crop them to the sub-region currently used.
|
125
|
+
At this stage, dataset_info.flats should be a dict in the form {num: array}
|
126
|
+
|
127
|
+
Parameters
|
128
|
+
----------
|
129
|
+
sub_region: 2-tuple of 3-tuples of int
|
130
|
+
Tuple in the form ((start_y, end_y), (start_x, end_x))
|
131
|
+
"""
|
132
|
+
(start_y, end_y), (start_x, end_x) = sub_region
|
133
|
+
|
134
|
+
processing_func_args = processing_func_args or []
|
135
|
+
processing_func_kwargs = processing_func_kwargs or {}
|
136
|
+
|
137
|
+
def proc(img):
|
138
|
+
if processing_func is None:
|
139
|
+
return img
|
140
|
+
return processing_func(img, *processing_func_args, **processing_func_kwargs)
|
141
|
+
|
142
|
+
res = {
|
143
|
+
"flats": {k: proc(flat_k)[start_y:end_y, start_x:end_x] for k, flat_k in dataset_info.flats.items()},
|
144
|
+
"darks": {k: proc(dark_k)[start_y:end_y, start_x:end_x] for k, dark_k in dataset_info.darks.items()},
|
145
|
+
}
|
146
|
+
return res
|
@@ -2,8 +2,8 @@ import os
|
|
2
2
|
import pytest
|
3
3
|
import numpy as np
|
4
4
|
from nabu.testutils import utilstest, __do_long_tests__
|
5
|
-
from nabu.resources.dataset_analyzer import HDF5DatasetAnalyzer
|
6
|
-
from nabu.resources.
|
5
|
+
from nabu.resources.dataset_analyzer import HDF5DatasetAnalyzer, analyze_dataset
|
6
|
+
from nabu.resources.nxflatfield import update_dataset_info_flats_darks
|
7
7
|
from nabu.resources.utils import extract_parameters
|
8
8
|
from nabu.pipeline.estimators import CompositeCOREstimator
|
9
9
|
from nabu.pipeline.config import parse_nabu_config_file
|
@@ -26,7 +26,8 @@ def bootstrap(request):
|
|
26
26
|
cls.cor_pix = 1321.625
|
27
27
|
cls.abs_tol = 0.0001
|
28
28
|
cls.dataset_info = HDF5DatasetAnalyzer(dataset_downloaded_path)
|
29
|
-
cls.
|
29
|
+
update_dataset_info_flats_darks(cls.dataset_info, True)
|
30
|
+
cls.cor_options = extract_parameters("side=300.0; near_width = 20.0", sep=";")
|
30
31
|
|
31
32
|
|
32
33
|
@pytest.mark.skipif(not (__do_long_tests__), reason="Need NABU_LONG_TESTS=1 for this test")
|
@@ -52,6 +53,7 @@ def bootstrap_bamboo_reduced(request):
|
|
52
53
|
conf_relpath = os.path.join("bamboo_reduced.conf")
|
53
54
|
conf_downloaded_path = utilstest.getfile(conf_relpath)
|
54
55
|
cls.ds_std = analyze_dataset(dataset_downloaded_path)
|
56
|
+
update_dataset_info_flats_darks(cls.ds_std, True)
|
55
57
|
cls.conf_std = parse_nabu_config_file(conf_downloaded_path)
|
56
58
|
|
57
59
|
# Dataset with estimated_cor_frm_motor
|
@@ -60,90 +62,60 @@ def bootstrap_bamboo_reduced(request):
|
|
60
62
|
conf_relpath = os.path.join("bamboo_reduced_bliss.conf")
|
61
63
|
conf_downloaded_path = utilstest.getfile(conf_relpath)
|
62
64
|
cls.ds_bliss = analyze_dataset(dataset_downloaded_path)
|
65
|
+
update_dataset_info_flats_darks(cls.ds_bliss, True)
|
63
66
|
cls.conf_bliss = parse_nabu_config_file(conf_downloaded_path)
|
64
67
|
|
65
68
|
|
66
69
|
@pytest.mark.skipif(not (__do_long_tests__), reason="need environment variable NABU_LONG_TESTS=1")
|
67
70
|
@pytest.mark.usefixtures("bootstrap_bamboo_reduced")
|
68
71
|
class TestCorNearPos:
|
69
|
-
|
72
|
+
# TODO adapt test file
|
73
|
+
true_cor = 339.486 - 0.5
|
70
74
|
|
71
75
|
def test_cor_sliding_standard(self):
|
72
76
|
cor_options = extract_parameters(self.conf_std["reconstruction"].get("cor_options", None), sep=";")
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
cor = finder.find_cor()
|
81
|
-
message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
|
82
|
-
assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
|
83
|
-
|
84
|
-
cor_options.update({"side": "center"})
|
85
|
-
finder = CORFinder("sliding-window", self.ds_std, do_flatfield=True, cor_options=cor_options)
|
86
|
-
cor = finder.find_cor()
|
87
|
-
message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
|
88
|
-
assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
|
77
|
+
for side in [None, "from_file", "center"]:
|
78
|
+
if side is not None:
|
79
|
+
cor_options.update({"side": side})
|
80
|
+
finder = CORFinder("sliding-window", self.ds_std, do_flatfield=True, cor_options=cor_options)
|
81
|
+
cor = finder.find_cor()
|
82
|
+
message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
|
83
|
+
assert np.isclose(self.true_cor, cor, atol=self.abs_tol + 0.5), message # FIXME
|
89
84
|
|
90
85
|
def test_cor_fourier_angles_standard(self):
|
91
86
|
cor_options = extract_parameters(self.conf_std["reconstruction"].get("cor_options", None), sep=";")
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
cor_options.update({"side": "center"})
|
105
|
-
finder = SinoCORFinder("fourier-angles", self.ds_std, do_flatfield=True, cor_options=cor_options)
|
106
|
-
cor = finder.find_cor()
|
107
|
-
message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
|
108
|
-
assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
|
87
|
+
# TODO modify test files
|
88
|
+
if "near_pos" in cor_options and "near" in cor_options.get("side", "") == "near":
|
89
|
+
cor_options["side"] = cor_options["near_pos"]
|
90
|
+
#
|
91
|
+
for side in [None, "from_file", "center"]:
|
92
|
+
if side is not None:
|
93
|
+
cor_options.update({"side": side})
|
94
|
+
finder = SinoCORFinder("fourier-angles", self.ds_std, do_flatfield=True, cor_options=cor_options)
|
95
|
+
cor = finder.find_cor()
|
96
|
+
message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
|
97
|
+
assert np.isclose(self.true_cor + 0.5, cor, atol=self.abs_tol), message
|
109
98
|
|
110
99
|
def test_cor_sliding_bliss(self):
|
111
100
|
cor_options = extract_parameters(self.conf_bliss["reconstruction"].get("cor_options", None), sep=";")
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
|
124
|
-
|
125
|
-
cor_options.update({"side": "center"})
|
126
|
-
finder = CORFinder("sliding-window", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
|
127
|
-
cor = finder.find_cor()
|
128
|
-
message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
|
129
|
-
assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
|
101
|
+
# TODO modify test files
|
102
|
+
if "near_pos" in cor_options and "near" in cor_options.get("side", "") == "near":
|
103
|
+
cor_options["side"] = cor_options["near_pos"]
|
104
|
+
#
|
105
|
+
for side in [None, "from_file", "center"]:
|
106
|
+
if side is not None:
|
107
|
+
cor_options.update({"side": side})
|
108
|
+
finder = CORFinder("sliding-window", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
|
109
|
+
cor = finder.find_cor()
|
110
|
+
message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
|
111
|
+
assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
|
130
112
|
|
131
113
|
def test_cor_fourier_angles_bliss(self):
|
132
114
|
cor_options = extract_parameters(self.conf_bliss["reconstruction"].get("cor_options", None), sep=";")
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
finder = SinoCORFinder("fourier-angles", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
|
141
|
-
cor = finder.find_cor()
|
142
|
-
message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
|
143
|
-
assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
|
144
|
-
|
145
|
-
cor_options.update({"side": "center"})
|
146
|
-
finder = SinoCORFinder("fourier-angles", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
|
147
|
-
cor = finder.find_cor()
|
148
|
-
message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
|
149
|
-
assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
|
115
|
+
for side in [None, "from_file", "center"]:
|
116
|
+
if side is not None:
|
117
|
+
cor_options.update({"side": side})
|
118
|
+
finder = SinoCORFinder("fourier-angles", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
|
119
|
+
cor = finder.find_cor()
|
120
|
+
message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
|
121
|
+
assert np.isclose(self.true_cor + 0.5, cor, atol=self.abs_tol), message
|
nabu/pipeline/utils.py
CHANGED
@@ -28,7 +28,7 @@ def pipeline_step(step_attr, step_desc):
|
|
28
28
|
def decorator(func):
|
29
29
|
def wrapper(*args, **kwargs):
|
30
30
|
self = args[0]
|
31
|
-
if self
|
31
|
+
if getattr(self, step_attr, None) is None:
|
32
32
|
return
|
33
33
|
self.logger.info(step_desc)
|
34
34
|
res = func(*args, **kwargs)
|
@@ -38,7 +38,9 @@ def pipeline_step(step_attr, step_desc):
|
|
38
38
|
for callback in callbacks:
|
39
39
|
callback(self)
|
40
40
|
if self.datadump_manager is not None and step_name in self.datadump_manager.data_dump:
|
41
|
-
self.datadump_manager.dump_data_to_file(
|
41
|
+
self.datadump_manager.dump_data_to_file(
|
42
|
+
step_name, self.radios, crop_margin=not (self._radios_were_cropped)
|
43
|
+
)
|
42
44
|
return res
|
43
45
|
|
44
46
|
return wrapper
|