nabu 2024.1.9__py3-none-any.whl → 2024.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nabu/__init__.py +1 -1
- nabu/app/bootstrap.py +2 -3
- nabu/app/cast_volume.py +4 -2
- nabu/app/cli_configs.py +5 -0
- nabu/app/composite_cor.py +1 -1
- nabu/app/create_distortion_map_from_poly.py +5 -6
- nabu/app/diag_to_pix.py +7 -19
- nabu/app/diag_to_rot.py +14 -29
- nabu/app/double_flatfield.py +32 -44
- nabu/app/parse_reconstruction_log.py +3 -0
- nabu/app/reconstruct.py +53 -15
- nabu/app/reconstruct_helical.py +2 -2
- nabu/app/stitching.py +27 -13
- nabu/app/tests/test_reduce_dark_flat.py +4 -1
- nabu/cuda/kernel.py +11 -2
- nabu/cuda/processing.py +2 -2
- nabu/cuda/src/cone.cu +77 -0
- nabu/cuda/src/hierarchical_backproj.cu +271 -0
- nabu/cuda/utils.py +0 -6
- nabu/estimation/alignment.py +5 -19
- nabu/estimation/cor.py +173 -599
- nabu/estimation/cor_sino.py +356 -26
- nabu/estimation/focus.py +63 -11
- nabu/estimation/tests/test_cor.py +124 -58
- nabu/estimation/tests/test_focus.py +6 -6
- nabu/estimation/tilt.py +2 -1
- nabu/estimation/utils.py +5 -33
- nabu/io/__init__.py +1 -1
- nabu/io/cast_volume.py +1 -1
- nabu/io/reader.py +416 -21
- nabu/io/tests/test_readers.py +422 -0
- nabu/io/tests/test_writers.py +1 -102
- nabu/io/writer.py +4 -433
- nabu/opencl/kernel.py +14 -3
- nabu/opencl/processing.py +8 -0
- nabu/pipeline/config_validators.py +5 -2
- nabu/pipeline/datadump.py +12 -5
- nabu/pipeline/estimators.py +162 -188
- nabu/pipeline/fullfield/chunked.py +168 -92
- nabu/pipeline/fullfield/chunked_cuda.py +7 -3
- nabu/pipeline/fullfield/computations.py +2 -7
- nabu/pipeline/fullfield/dataset_validator.py +0 -4
- nabu/pipeline/fullfield/nabu_config.py +37 -13
- nabu/pipeline/fullfield/processconfig.py +22 -13
- nabu/pipeline/fullfield/reconstruction.py +13 -9
- nabu/pipeline/helical/helical_chunked_regridded.py +1 -1
- nabu/pipeline/helical/helical_chunked_regridded_cuda.py +1 -0
- nabu/pipeline/helical/helical_reconstruction.py +1 -1
- nabu/pipeline/params.py +21 -1
- nabu/pipeline/processconfig.py +1 -12
- nabu/pipeline/reader.py +146 -0
- nabu/pipeline/tests/test_estimators.py +44 -72
- nabu/pipeline/utils.py +4 -2
- nabu/pipeline/writer.py +10 -2
- nabu/preproc/ccd_cuda.py +1 -1
- nabu/preproc/ctf.py +14 -7
- nabu/preproc/ctf_cuda.py +2 -3
- nabu/preproc/double_flatfield.py +5 -12
- nabu/preproc/double_flatfield_cuda.py +2 -2
- nabu/preproc/flatfield.py +5 -1
- nabu/preproc/flatfield_cuda.py +5 -1
- nabu/preproc/phase.py +24 -73
- nabu/preproc/phase_cuda.py +5 -8
- nabu/preproc/tests/test_ctf.py +11 -7
- nabu/preproc/tests/test_flatfield.py +67 -122
- nabu/preproc/tests/test_paganin.py +54 -30
- nabu/processing/azim.py +206 -0
- nabu/processing/convolution_cuda.py +1 -1
- nabu/processing/fft_cuda.py +15 -17
- nabu/processing/histogram.py +2 -0
- nabu/processing/histogram_cuda.py +2 -1
- nabu/processing/kernel_base.py +3 -0
- nabu/processing/muladd_cuda.py +1 -0
- nabu/processing/padding_opencl.py +1 -1
- nabu/processing/roll_opencl.py +1 -0
- nabu/processing/rotation_cuda.py +2 -2
- nabu/processing/tests/test_fft.py +17 -10
- nabu/processing/unsharp_cuda.py +1 -1
- nabu/reconstruction/cone.py +104 -40
- nabu/reconstruction/fbp.py +3 -0
- nabu/reconstruction/fbp_base.py +7 -2
- nabu/reconstruction/filtering.py +20 -7
- nabu/reconstruction/filtering_cuda.py +7 -1
- nabu/reconstruction/hbp.py +424 -0
- nabu/reconstruction/mlem.py +99 -0
- nabu/reconstruction/reconstructor.py +2 -0
- nabu/reconstruction/rings_cuda.py +19 -19
- nabu/reconstruction/sinogram_cuda.py +1 -0
- nabu/reconstruction/sinogram_opencl.py +3 -1
- nabu/reconstruction/tests/test_cone.py +10 -5
- nabu/reconstruction/tests/test_deringer.py +7 -6
- nabu/reconstruction/tests/test_fbp.py +124 -10
- nabu/reconstruction/tests/test_filtering.py +13 -11
- nabu/reconstruction/tests/test_halftomo.py +30 -4
- nabu/reconstruction/tests/test_mlem.py +91 -0
- nabu/reconstruction/tests/test_reconstructor.py +8 -3
- nabu/resources/dataset_analyzer.py +142 -92
- nabu/resources/gpu.py +1 -0
- nabu/resources/nxflatfield.py +134 -125
- nabu/resources/templates/id16a_fluo.conf +42 -0
- nabu/resources/tests/test_extract.py +10 -0
- nabu/resources/tests/test_nxflatfield.py +2 -2
- nabu/stitching/alignment.py +80 -24
- nabu/stitching/config.py +105 -68
- nabu/stitching/definitions.py +1 -0
- nabu/stitching/frame_composition.py +68 -60
- nabu/stitching/overlap.py +91 -51
- nabu/stitching/single_axis_stitching.py +32 -0
- nabu/stitching/slurm_utils.py +6 -6
- nabu/stitching/stitcher/__init__.py +0 -0
- nabu/stitching/stitcher/base.py +124 -0
- nabu/stitching/stitcher/dumper/__init__.py +3 -0
- nabu/stitching/stitcher/dumper/base.py +94 -0
- nabu/stitching/stitcher/dumper/postprocessing.py +356 -0
- nabu/stitching/stitcher/dumper/preprocessing.py +60 -0
- nabu/stitching/stitcher/post_processing.py +555 -0
- nabu/stitching/stitcher/pre_processing.py +1068 -0
- nabu/stitching/stitcher/single_axis.py +484 -0
- nabu/stitching/stitcher/stitcher.py +0 -0
- nabu/stitching/stitcher/y_stitcher.py +13 -0
- nabu/stitching/stitcher/z_stitcher.py +45 -0
- nabu/stitching/stitcher_2D.py +278 -0
- nabu/stitching/tests/test_config.py +12 -37
- nabu/stitching/tests/test_frame_composition.py +33 -59
- nabu/stitching/tests/test_overlap.py +149 -7
- nabu/stitching/tests/test_utils.py +1 -1
- nabu/stitching/tests/test_y_preprocessing_stitching.py +132 -0
- nabu/stitching/tests/{test_z_stitching.py → test_z_postprocessing_stitching.py} +167 -561
- nabu/stitching/tests/test_z_preprocessing_stitching.py +431 -0
- nabu/stitching/utils/__init__.py +1 -0
- nabu/stitching/utils/post_processing.py +281 -0
- nabu/stitching/utils/tests/test_post-processing.py +21 -0
- nabu/stitching/{utils.py → utils/utils.py} +79 -52
- nabu/stitching/y_stitching.py +27 -0
- nabu/stitching/z_stitching.py +32 -2263
- nabu/testutils.py +1 -152
- nabu/thirdparty/tomocupy_remove_stripe.py +43 -9
- nabu/utils.py +158 -61
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/METADATA +10 -3
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/RECORD +144 -121
- nabu/io/tiffwriter_zmm.py +0 -99
- nabu/pipeline/fallback_utils.py +0 -149
- nabu/pipeline/helical/tests/test_accumulator.py +0 -158
- nabu/pipeline/helical/tests/test_pipeline_elements_full.py +0 -355
- nabu/pipeline/helical/tests/test_strategy.py +0 -61
- nabu/pipeline/helical/utils.py +0 -51
- nabu/pipeline/tests/test_chunk_reader.py +0 -74
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/LICENSE +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/WHEEL +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/entry_points.txt +0 -0
- {nabu-2024.1.9.dist-info → nabu-2024.2.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,422 @@
|
|
1
|
+
from math import ceil
|
2
|
+
from tempfile import TemporaryDirectory
|
3
|
+
from dataclasses import dataclass
|
4
|
+
from tomoscan.io import HDF5File
|
5
|
+
import pytest
|
6
|
+
import numpy as np
|
7
|
+
from nxtomo.application.nxtomo import ImageKey
|
8
|
+
from tomoscan.esrf import EDFVolume
|
9
|
+
from nabu.pipeline.reader import NXTomoReaderBinning
|
10
|
+
from nabu.testutils import utilstest, __do_long_tests__, get_file
|
11
|
+
from nabu.utils import indices_to_slices, merge_slices
|
12
|
+
from nabu.io.reader import EDFStackReader, NXTomoReader, NXDarksFlats
|
13
|
+
|
14
|
+
|
15
|
+
@dataclass
|
16
|
+
class SimpleNXTomoDescription:
|
17
|
+
n_darks: int = 0
|
18
|
+
n_flats1: int = 0
|
19
|
+
n_projs: int = 0
|
20
|
+
n_flats2: int = 0
|
21
|
+
n_align: int = 0
|
22
|
+
frame_shape: tuple = None
|
23
|
+
dtype: np.dtype = np.uint16
|
24
|
+
|
25
|
+
|
26
|
+
@pytest.fixture(scope="class")
|
27
|
+
def bootstrap_nx_reader(request):
|
28
|
+
cls = request.cls
|
29
|
+
|
30
|
+
cls.nx_fname = utilstest.getfile("dummy_nxtomo.nx")
|
31
|
+
cls.nx_data_path = "entry/instrument/detector/data"
|
32
|
+
cls.data_desc = SimpleNXTomoDescription(
|
33
|
+
n_darks=10, n_flats1=11, n_projs=100, n_flats2=11, n_align=12, frame_shape=(11, 10), dtype=np.uint16
|
34
|
+
)
|
35
|
+
cls.projs_vals = np.arange(cls.data_desc.n_projs) + cls.data_desc.n_flats1 + cls.data_desc.n_darks
|
36
|
+
cls.darks_vals = np.arange(cls.data_desc.n_darks)
|
37
|
+
cls.flats1_vals = np.arange(cls.data_desc.n_darks, cls.data_desc.n_darks + cls.data_desc.n_flats1)
|
38
|
+
cls.flats2_vals = np.arange(cls.data_desc.n_darks, cls.data_desc.n_darks + cls.data_desc.n_flats2)
|
39
|
+
|
40
|
+
yield
|
41
|
+
# teardown
|
42
|
+
|
43
|
+
|
44
|
+
@pytest.mark.usefixtures("bootstrap_nx_reader")
|
45
|
+
class TestNXReader:
|
46
|
+
def test_incorrect_path(self):
|
47
|
+
with pytest.raises(FileNotFoundError):
|
48
|
+
reader = NXTomoReader("/invalid/path", self.nx_data_path)
|
49
|
+
with pytest.raises(KeyError):
|
50
|
+
reader = NXTomoReader(self.nx_fname, "/bad/data/path")
|
51
|
+
|
52
|
+
def test_simple_reads(self):
|
53
|
+
"""
|
54
|
+
Test NXTomoReader with simplest settings
|
55
|
+
"""
|
56
|
+
reader1 = NXTomoReader(self.nx_fname, self.nx_data_path)
|
57
|
+
data1 = reader1.load_data()
|
58
|
+
assert data1.shape == (self.data_desc.n_projs,) + self.data_desc.frame_shape
|
59
|
+
assert np.allclose(data1[:, 0, 0], self.projs_vals)
|
60
|
+
|
61
|
+
def test_image_key(self):
|
62
|
+
"""
|
63
|
+
Test the data selection using "image_key".
|
64
|
+
"""
|
65
|
+
reader_projs = NXTomoReader(self.nx_fname, self.nx_data_path, image_key=ImageKey.PROJECTION.value)
|
66
|
+
data = reader_projs.load_data()
|
67
|
+
assert np.allclose(data[:, 0, 0], self.projs_vals)
|
68
|
+
|
69
|
+
reader_darks = NXTomoReader(self.nx_fname, self.nx_data_path, image_key=ImageKey.DARK_FIELD.value)
|
70
|
+
data_darks = reader_darks.load_data()
|
71
|
+
assert np.allclose(data_darks[:, 0, 0], self.darks_vals)
|
72
|
+
|
73
|
+
reader_flats = NXTomoReader(self.nx_fname, self.nx_data_path, image_key=ImageKey.FLAT_FIELD.value)
|
74
|
+
data_flats = reader_flats.load_data()
|
75
|
+
assert np.allclose(data_flats[:, 0, 0], np.concatenate([self.flats1_vals, self.flats2_vals]))
|
76
|
+
|
77
|
+
def test_data_buffer_and_subregion(self):
|
78
|
+
"""
|
79
|
+
Test the "data_buffer" and "sub_region" parameters
|
80
|
+
"""
|
81
|
+
data_desc = self.data_desc
|
82
|
+
|
83
|
+
def _check_correct_shape_succeeds(shape, sub_region, test_description=""):
|
84
|
+
err_msg = "Something wrong with the following test:" + test_description
|
85
|
+
data_buffer = np.zeros(shape, dtype="f")
|
86
|
+
reader1 = NXTomoReader(self.nx_fname, self.nx_data_path, sub_region=sub_region)
|
87
|
+
data1 = reader1.load_data(output=data_buffer)
|
88
|
+
assert id(data1) == id(data_buffer), err_msg
|
89
|
+
reader2 = NXTomoReader(self.nx_fname, self.nx_data_path, sub_region=sub_region)
|
90
|
+
data2 = reader2.load_data()
|
91
|
+
assert np.allclose(data1, data2), err_msg
|
92
|
+
|
93
|
+
test_cases = [
|
94
|
+
{
|
95
|
+
"description": "In the projections, read everything into the provided data buffer",
|
96
|
+
"sub_region": None,
|
97
|
+
"correct_shape": (data_desc.n_projs,) + data_desc.frame_shape,
|
98
|
+
"wrong_shapes": [
|
99
|
+
(data_desc.n_projs - 1,) + data_desc.frame_shape,
|
100
|
+
(data_desc.n_projs - 1,) + (999, 998),
|
101
|
+
(data_desc.n_projs,) + (999, 998),
|
102
|
+
],
|
103
|
+
},
|
104
|
+
{
|
105
|
+
"description": "In the projections, select a subset along dimension 0 (i.e take only several full frames). The correct output shape is: data_total[image_key==0][slice(10, 30)].shape",
|
106
|
+
"sub_region": slice(10, 30),
|
107
|
+
"correct_shape": (20,) + data_desc.frame_shape,
|
108
|
+
"wrong_shapes": [
|
109
|
+
(data_desc.n_projs,) + data_desc.frame_shape,
|
110
|
+
(19,) + data_desc.frame_shape,
|
111
|
+
],
|
112
|
+
},
|
113
|
+
{
|
114
|
+
"description": "In the projections, read several rows of all images, i.e extract several sinograms. The correct output shape is: data_total[image_key==0][:, slice(start_z, end_z), :].shape",
|
115
|
+
"sub_region": (None, slice(3, 7), None),
|
116
|
+
"correct_shape": (data_desc.n_projs, 4, data_desc.frame_shape[-1]),
|
117
|
+
"wrong_shapes": [],
|
118
|
+
},
|
119
|
+
]
|
120
|
+
|
121
|
+
for test_case in test_cases:
|
122
|
+
for wrong_shape in test_case["wrong_shapes"]:
|
123
|
+
with pytest.raises(ValueError):
|
124
|
+
data_buffer_wrong_shape = np.zeros(wrong_shape, dtype="f")
|
125
|
+
reader = NXTomoReader(
|
126
|
+
self.nx_fname,
|
127
|
+
self.nx_data_path,
|
128
|
+
sub_region=test_case["sub_region"],
|
129
|
+
)
|
130
|
+
reader.load_data(output=data_buffer_wrong_shape)
|
131
|
+
_check_correct_shape_succeeds(test_case["correct_shape"], test_case["sub_region"], test_case["description"])
|
132
|
+
|
133
|
+
def test_subregion_and_subsampling(self):
|
134
|
+
data_desc = self.data_desc
|
135
|
+
test_cases = [
|
136
|
+
{
|
137
|
+
# Read one full image out of two in all projections
|
138
|
+
"sub_region": (slice(None, None, 2), None, None),
|
139
|
+
"expected_shape": (self.projs_vals[::2].size,) + data_desc.frame_shape,
|
140
|
+
"expected_values": self.projs_vals[::2],
|
141
|
+
},
|
142
|
+
{
|
143
|
+
# Read one image fragment (several rows) out of two in all projections
|
144
|
+
"sub_region": (slice(None, None, 2), slice(5, 8), None),
|
145
|
+
"expected_shape": (self.projs_vals[::2].size, 3, data_desc.frame_shape[-1]),
|
146
|
+
"expected_values": self.projs_vals[::2],
|
147
|
+
},
|
148
|
+
]
|
149
|
+
|
150
|
+
for test_case in test_cases:
|
151
|
+
reader = NXTomoReader(self.nx_fname, self.nx_data_path, sub_region=test_case["sub_region"])
|
152
|
+
data = reader.load_data()
|
153
|
+
assert data.shape == test_case["expected_shape"]
|
154
|
+
assert np.allclose(data[:, 0, 0], test_case["expected_values"])
|
155
|
+
|
156
|
+
def test_reading_with_binning_(self):
|
157
|
+
from nabu.pipeline.reader import NXTomoReaderBinning
|
158
|
+
|
159
|
+
reader_with_binning = NXTomoReaderBinning((2, 2), self.nx_fname, self.nx_data_path)
|
160
|
+
data = reader_with_binning.load_data()
|
161
|
+
assert data.shape == (self.data_desc.n_projs,) + tuple(n // 2 for n in self.data_desc.frame_shape)
|
162
|
+
|
163
|
+
def test_reading_with_distortion_correction(self):
|
164
|
+
from nabu.io.detector_distortion import DetectorDistortionBase
|
165
|
+
from nabu.pipeline.reader import NXTomoReaderDistortionCorrection
|
166
|
+
|
167
|
+
data_desc = self.data_desc
|
168
|
+
|
169
|
+
# (start_x, end_x, start_y, end_y)
|
170
|
+
sub_region_xy = (None, None, 1, 6)
|
171
|
+
|
172
|
+
distortion_corrector = DetectorDistortionBase(detector_full_shape_vh=data_desc.frame_shape)
|
173
|
+
distortion_corrector.set_sub_region_transformation(target_sub_region=sub_region_xy)
|
174
|
+
adapted_subregion = distortion_corrector.get_adapted_subregion(sub_region_xy)
|
175
|
+
sub_region = (slice(None, None), slice(*sub_region_xy[2:]), slice(*sub_region_xy[:2]))
|
176
|
+
|
177
|
+
reader_distortion_corr = NXTomoReaderDistortionCorrection(
|
178
|
+
distortion_corrector,
|
179
|
+
self.nx_fname,
|
180
|
+
self.nx_data_path,
|
181
|
+
sub_region=sub_region,
|
182
|
+
)
|
183
|
+
|
184
|
+
reader_distortion_corr.load_data()
|
185
|
+
|
186
|
+
@pytest.mark.skipif(not (__do_long_tests__), reason="Need NABU_LONG_TESTS=1")
|
187
|
+
def test_other_load_patterns(self):
|
188
|
+
"""
|
189
|
+
Other data read patterns that are sometimes used by ChunkedPipeline
|
190
|
+
Test cases already done in check_correct_shape_succeeds():
|
191
|
+
- Read all frames in a provided buffer
|
192
|
+
- Read a subset of all (full) projections
|
193
|
+
- Read several rows of all projections (extract sinograms)
|
194
|
+
"""
|
195
|
+
data_desc = self.data_desc
|
196
|
+
|
197
|
+
test_cases = [
|
198
|
+
{
|
199
|
+
"description": "Select a subset along all dimensions. The correct output shape is data_total[image_key==0][slice_dim0, slice_dim1, slice_dim2].shape",
|
200
|
+
"sub_region": (slice(10, 72, 2), slice(4, None), slice(2, 8)),
|
201
|
+
"expected_shape": (31, 7, 6),
|
202
|
+
"expected_values": self.projs_vals[slice(10, 72, 2)],
|
203
|
+
},
|
204
|
+
{
|
205
|
+
"description": "Select several rows in all images (i.e extract sinograms), with binning",
|
206
|
+
"sub_region": (slice(None, None), slice(3, 7), slice(None, None)),
|
207
|
+
"binning": (2, 2),
|
208
|
+
"expected_shape": (data_desc.n_projs, 4 // 2, data_desc.frame_shape[-1] // 2),
|
209
|
+
"expected_values": self.projs_vals[:],
|
210
|
+
},
|
211
|
+
{
|
212
|
+
"description": "Extract sinograms with binning + subsampling",
|
213
|
+
"sub_region": (slice(None, None, 2), slice(1, 8), slice(None, None)),
|
214
|
+
"binning": (2, 2),
|
215
|
+
"expected_shape": (ceil(data_desc.n_projs / 2), 7 // 2, data_desc.frame_shape[-1] // 2),
|
216
|
+
"expected_values": self.projs_vals[::2],
|
217
|
+
},
|
218
|
+
]
|
219
|
+
|
220
|
+
for test_case in test_cases:
|
221
|
+
binning = test_case.get("binning", None)
|
222
|
+
reader_cls = NXTomoReader
|
223
|
+
init_args = [self.nx_fname, self.nx_data_path]
|
224
|
+
init_kwargs = {"sub_region": test_case["sub_region"]}
|
225
|
+
if binning is not None:
|
226
|
+
reader_cls = NXTomoReaderBinning
|
227
|
+
init_args = [binning] + init_args
|
228
|
+
reader = reader_cls(*init_args, **init_kwargs)
|
229
|
+
data = reader.load_data()
|
230
|
+
err_msg = "Something wrong with test: " + test_case["description"]
|
231
|
+
assert data.shape == test_case["expected_shape"], err_msg
|
232
|
+
assert np.allclose(data[:, 0, 0], test_case["expected_values"]), err_msg
|
233
|
+
|
234
|
+
|
235
|
+
@pytest.fixture(scope="class")
|
236
|
+
def bootstrap_edf_reader(request):
|
237
|
+
cls = request.cls
|
238
|
+
|
239
|
+
test_dir = utilstest.data_home
|
240
|
+
cls._tmpdir = TemporaryDirectory(prefix="test_edf_stack_", dir=test_dir)
|
241
|
+
cls.edf_dir = cls._tmpdir.name
|
242
|
+
cls.n_projs = 100
|
243
|
+
cls.frame_shape = (11, 12)
|
244
|
+
cls.projs_vals = np.arange(cls.n_projs, dtype=np.uint16) + 10
|
245
|
+
|
246
|
+
edf_vol = EDFVolume(folder=cls.edf_dir, volume_basename="edf_stack", overwrite=True)
|
247
|
+
data_shape = (cls.n_projs,) + cls.frame_shape
|
248
|
+
edf_vol.data = np.ones(data_shape, dtype=np.uint16) * cls.projs_vals.reshape(cls.n_projs, 1, 1)
|
249
|
+
edf_vol.save_data()
|
250
|
+
cls.filenames = list(edf_vol.browse_data_files())
|
251
|
+
|
252
|
+
yield
|
253
|
+
cls._tmpdir.cleanup()
|
254
|
+
|
255
|
+
|
256
|
+
@pytest.mark.usefixtures("bootstrap_edf_reader")
|
257
|
+
class TestEDFReader:
|
258
|
+
def test_read_all_frames(self):
|
259
|
+
"""
|
260
|
+
Simple test, read all the frames
|
261
|
+
"""
|
262
|
+
reader = EDFStackReader(self.filenames)
|
263
|
+
data = reader.load_data()
|
264
|
+
expected_shape = (self.n_projs,) + self.frame_shape
|
265
|
+
assert data.shape == expected_shape
|
266
|
+
assert np.allclose(data[:, 0, 0], self.projs_vals)
|
267
|
+
|
268
|
+
buffer_correct = np.zeros(expected_shape, dtype=np.float32)
|
269
|
+
reader.load_data(output=buffer_correct)
|
270
|
+
|
271
|
+
buffer_incorrect_1 = np.zeros((99, 11, 12), dtype=np.float32)
|
272
|
+
with pytest.raises(ValueError):
|
273
|
+
reader.load_data(output=buffer_incorrect_1)
|
274
|
+
|
275
|
+
buffer_incorrect_2 = np.zeros((100, 11, 12), dtype=np.uint16)
|
276
|
+
with pytest.raises(ValueError):
|
277
|
+
reader.load_data(output=buffer_incorrect_2)
|
278
|
+
|
279
|
+
def test_subregions_1(self):
|
280
|
+
test_cases = [
|
281
|
+
{
|
282
|
+
"name": "read a handful of full frames",
|
283
|
+
"sub_region": (slice(0, 48), slice(None, None), slice(None, None)),
|
284
|
+
"expected_shape": (48,) + self.frame_shape,
|
285
|
+
"expected_values": self.projs_vals[:48],
|
286
|
+
},
|
287
|
+
{
|
288
|
+
"name": "read several lines of all frames (i.e extract a singoram)",
|
289
|
+
"sub_region": (slice(None, None), slice(0, 6), slice(None, None)),
|
290
|
+
"expected_shape": (self.n_projs, 6, self.frame_shape[-1]),
|
291
|
+
"expected_values": self.projs_vals,
|
292
|
+
},
|
293
|
+
{
|
294
|
+
"name": "read several lines of all frames (i.e extract a singoram), and a X-ROI",
|
295
|
+
"sub_region": (slice(None, None), slice(3, 7), slice(2, 5)),
|
296
|
+
"expected_shape": (self.n_projs, 4, 3),
|
297
|
+
"expected_values": self.projs_vals,
|
298
|
+
},
|
299
|
+
{
|
300
|
+
"name": "read several lines of all frames (i.e extract a singoram), with angular subsampling",
|
301
|
+
"sub_region": (slice(None, None, 2), slice(3, 7), slice(2, 5)),
|
302
|
+
"expected_shape": (ceil(self.n_projs / 2), 4, 3),
|
303
|
+
"expected_values": self.projs_vals[::2],
|
304
|
+
},
|
305
|
+
]
|
306
|
+
for test_case in test_cases:
|
307
|
+
reader = EDFStackReader(self.filenames, sub_region=test_case["sub_region"])
|
308
|
+
data = reader.load_data()
|
309
|
+
err_msg = "Something wrong with test: %s" % (test_case["name"])
|
310
|
+
assert data.shape == test_case["expected_shape"], err_msg
|
311
|
+
assert np.allclose(data[:, 0, 0], test_case["expected_values"]), err_msg
|
312
|
+
|
313
|
+
@pytest.mark.skipif(not (__do_long_tests__), reason="Need NABU_LONG_TESTS=1")
|
314
|
+
def test_reading_with_binning(self):
|
315
|
+
from nabu.pipeline.reader import EDFStackReaderBinning
|
316
|
+
|
317
|
+
reader_with_binning = EDFStackReaderBinning((2, 2), self.filenames)
|
318
|
+
data = reader_with_binning.load_data()
|
319
|
+
assert data.shape == (self.n_projs,) + tuple(n // 2 for n in self.frame_shape)
|
320
|
+
|
321
|
+
@pytest.mark.skipif(not (__do_long_tests__), reason="Need NABU_LONG_TESTS=1")
|
322
|
+
def test_reading_with_distortion_correction(self):
|
323
|
+
from nabu.io.detector_distortion import DetectorDistortionBase
|
324
|
+
from nabu.pipeline.reader import EDFStackReaderDistortionCorrection
|
325
|
+
|
326
|
+
# (start_x, end_x, start_y, end_y)
|
327
|
+
sub_region_xy = (None, None, 1, 6)
|
328
|
+
|
329
|
+
distortion_corrector = DetectorDistortionBase(detector_full_shape_vh=self.frame_shape)
|
330
|
+
distortion_corrector.set_sub_region_transformation(target_sub_region=sub_region_xy)
|
331
|
+
adapted_subregion = distortion_corrector.get_adapted_subregion(sub_region_xy)
|
332
|
+
sub_region = (slice(None, None), slice(*sub_region_xy[2:]), slice(*sub_region_xy[:2]))
|
333
|
+
|
334
|
+
reader_distortion_corr = EDFStackReaderDistortionCorrection(
|
335
|
+
distortion_corrector,
|
336
|
+
self.filenames,
|
337
|
+
sub_region=sub_region,
|
338
|
+
)
|
339
|
+
|
340
|
+
reader_distortion_corr.load_data()
|
341
|
+
|
342
|
+
|
343
|
+
def test_indices_to_slices():
|
344
|
+
slices1 = [slice(0, 4)]
|
345
|
+
slices2 = [slice(11, 16)]
|
346
|
+
slices3 = [slice(3, 5), slice(8, 20)]
|
347
|
+
slices4 = [slice(2, 7), slice(18, 28), slice(182, 845)]
|
348
|
+
idx = np.arange(1000)
|
349
|
+
for slices in [slices1, slices2, slices3, slices4]:
|
350
|
+
indices = np.hstack([idx[sl] for sl in slices])
|
351
|
+
slices_calculated = indices_to_slices(indices)
|
352
|
+
assert slices_calculated == slices, "Expected indices_to_slices() to return %s, but got %s" % (
|
353
|
+
str(slices),
|
354
|
+
str(slices_calculated),
|
355
|
+
)
|
356
|
+
|
357
|
+
|
358
|
+
def test_merge_slices():
|
359
|
+
idx = np.arange(10000)
|
360
|
+
rnd = lambda x: np.random.randint(1, high=x)
|
361
|
+
|
362
|
+
n_tests = 10
|
363
|
+
for i in range(n_tests):
|
364
|
+
start1 = rnd(1000)
|
365
|
+
stop1 = start1 + rnd(1000)
|
366
|
+
start2 = rnd(1000)
|
367
|
+
stop2 = start2 + rnd(1000)
|
368
|
+
step1 = rnd(4)
|
369
|
+
step2 = rnd(4)
|
370
|
+
slice1 = slice(start1, stop1, step1)
|
371
|
+
slice2 = slice(start2, stop2, step2)
|
372
|
+
|
373
|
+
assert np.allclose(idx[slice1][slice2], idx[merge_slices(slice1, slice2)])
|
374
|
+
|
375
|
+
|
376
|
+
@pytest.fixture(scope="class")
|
377
|
+
def bootstrap_nxdkrf(request):
|
378
|
+
cls = request.cls
|
379
|
+
|
380
|
+
cls.nx_file_path = get_file("bamboo_reduced.nx")
|
381
|
+
|
382
|
+
yield
|
383
|
+
# teardown
|
384
|
+
|
385
|
+
|
386
|
+
@pytest.mark.usefixtures("bootstrap_nxdkrf")
|
387
|
+
class TestDKRFReader:
|
388
|
+
def test_darks(self):
|
389
|
+
dkrf_reader = NXDarksFlats(self.nx_file_path)
|
390
|
+
darks = dkrf_reader.get_raw_darks(as_multiple_array=True)
|
391
|
+
reduced_darks = dkrf_reader.get_reduced_darks(method="mean")
|
392
|
+
|
393
|
+
actual_darks = []
|
394
|
+
with HDF5File(self.nx_file_path, "r") as f:
|
395
|
+
actual_darks.append(f["entry0000/data/data"][slice(0, 1)])
|
396
|
+
|
397
|
+
assert len(darks) == len(actual_darks)
|
398
|
+
|
399
|
+
for i in range(len(darks)):
|
400
|
+
assert np.allclose(darks[i], actual_darks[i])
|
401
|
+
actual_reduced_darks = np.mean(actual_darks[i], axis=0)
|
402
|
+
assert np.allclose(reduced_darks[i], actual_reduced_darks)
|
403
|
+
|
404
|
+
assert np.allclose(list(dkrf_reader.get_reduced_darks(as_dict=True).keys()), [0])
|
405
|
+
|
406
|
+
def test_flats(self):
|
407
|
+
dkrf_reader = NXDarksFlats(self.nx_file_path)
|
408
|
+
flats = dkrf_reader.get_raw_flats(as_multiple_array=True)
|
409
|
+
reduced_flats = dkrf_reader.get_reduced_flats(method="median")
|
410
|
+
|
411
|
+
actual_flats = []
|
412
|
+
with HDF5File(self.nx_file_path, "r") as f:
|
413
|
+
actual_flats.append(f["entry0000/data/data"][slice(1, 25 + 1)])
|
414
|
+
actual_flats.append(f["entry0000/data/data"][slice(526, 550 + 1)])
|
415
|
+
|
416
|
+
assert len(flats) == len(actual_flats)
|
417
|
+
for i in range(len(flats)):
|
418
|
+
assert np.allclose(flats[i], actual_flats[i])
|
419
|
+
actual_reduced_flats = np.median(actual_flats[i], axis=0)
|
420
|
+
assert np.allclose(reduced_flats[i], actual_reduced_flats)
|
421
|
+
|
422
|
+
assert np.allclose(list(dkrf_reader.get_reduced_flats(as_dict=True).keys()), [1, 526])
|
nabu/io/tests/test_writers.py
CHANGED
@@ -2,16 +2,10 @@ from os import path
|
|
2
2
|
from tempfile import TemporaryDirectory
|
3
3
|
import pytest
|
4
4
|
import numpy as np
|
5
|
-
from
|
6
|
-
from silx.io.dictdump import h5todict, dicttoh5
|
7
|
-
from nabu.misc.utils import psnr
|
8
|
-
from nabu.io.writer import NXProcessWriter, TIFFWriter, JP2Writer, __have_jp2k__
|
5
|
+
from nabu.io.writer import NXProcessWriter
|
9
6
|
from nabu.io.reader import import_h5_to_dict
|
10
7
|
from nabu.testutils import get_data
|
11
8
|
|
12
|
-
if __have_jp2k__:
|
13
|
-
from glymur import Jp2k
|
14
|
-
|
15
9
|
|
16
10
|
@pytest.fixture(scope="class")
|
17
11
|
def bootstrap(request):
|
@@ -25,101 +19,6 @@ def bootstrap(request):
|
|
25
19
|
cls._tmpdir.cleanup()
|
26
20
|
|
27
21
|
|
28
|
-
@pytest.mark.usefixtures("bootstrap")
|
29
|
-
class TestTiff:
|
30
|
-
def _check_tif_file(self, fname, expected_data, n_expected_images):
|
31
|
-
with TiffReader(fname) as tif:
|
32
|
-
assert len(tif.pages) == n_expected_images
|
33
|
-
for i in range(n_expected_images):
|
34
|
-
data_read = tif.pages[i].asarray()
|
35
|
-
if expected_data.ndim == 3:
|
36
|
-
expected_data_ = expected_data[i]
|
37
|
-
else:
|
38
|
-
expected_data_ = expected_data
|
39
|
-
assert np.allclose(data_read, expected_data_)
|
40
|
-
|
41
|
-
def test_2D(self):
|
42
|
-
# TODO use start_index=None (by default) in TIFFWriter
|
43
|
-
# if start_index is None:
|
44
|
-
# url = DataUrl(file_path=dirname(self.fname), data_path="basename(self.fname).tiff", scheme="tifffile")
|
45
|
-
# volume = TIFFVolume(volume_basename=basename(self.fname), data_url=url)
|
46
|
-
# volume.data = img
|
47
|
-
# volume.save()
|
48
|
-
pytest.skip("Writing a single 2D tiff is disabled for now")
|
49
|
-
|
50
|
-
fname = path.join(self.tempdir, "test_tiff2D.tif")
|
51
|
-
data = np.arange(100 * 101, dtype="f").reshape((100, 101))
|
52
|
-
nabu_tif = TIFFWriter(fname)
|
53
|
-
nabu_tif.write(data)
|
54
|
-
self._check_tif_file(fname, data, 1)
|
55
|
-
|
56
|
-
def test_3D_data_split_in_multiple_files(self):
|
57
|
-
fname = path.join(self.tempdir, "test_tiff3D_single.tif")
|
58
|
-
data = np.arange(11 * 100 * 101, dtype="f").reshape((11, 100, 101))
|
59
|
-
nabu_tif = TIFFWriter(fname, multiframe=False, start_index=500)
|
60
|
-
nabu_tif.write(data)
|
61
|
-
|
62
|
-
assert not (path.isfile(fname)), "found %s" % fname
|
63
|
-
|
64
|
-
prefix, ext = path.splitext(fname)
|
65
|
-
for i in range(data.shape[0]):
|
66
|
-
curr_rel_fname = prefix + str("_%06d" % (i + nabu_tif.start_index)) + ext
|
67
|
-
curr_fname = path.join(self.tempdir, curr_rel_fname)
|
68
|
-
self._check_tif_file(curr_fname, data[i], 1)
|
69
|
-
|
70
|
-
def test_3D_data_in_one_file(self):
|
71
|
-
fname = path.join(self.tempdir, "test_tiff3D_multi.tif")
|
72
|
-
data = np.arange(11 * 100 * 101, dtype="f").reshape((11, 100, 101))
|
73
|
-
nabu_tif = TIFFWriter(fname, multiframe=True)
|
74
|
-
nabu_tif.write(data)
|
75
|
-
|
76
|
-
assert path.isfile(fname)
|
77
|
-
self._check_tif_file(fname, data, data.shape[0])
|
78
|
-
|
79
|
-
|
80
|
-
@pytest.mark.skipif(not (__have_jp2k__), reason="Need openjpeg2000/glymur for this test")
|
81
|
-
@pytest.mark.usefixtures("bootstrap")
|
82
|
-
class TestJP2:
|
83
|
-
def _check_jp2_file(self, fname, expected_data, expected_psnr=None):
|
84
|
-
data = Jp2k(fname)[:]
|
85
|
-
if expected_psnr is None:
|
86
|
-
assert np.allclose(data, expected_data)
|
87
|
-
else:
|
88
|
-
computed_psnr = psnr(data, expected_data)
|
89
|
-
assert np.abs(computed_psnr - expected_psnr) < 1
|
90
|
-
|
91
|
-
def test_2D_lossless(self):
|
92
|
-
data = get_data("mri_sino500.npz")["data"].astype(np.uint16)
|
93
|
-
fname = path.join(self.tempdir, "sino500.jp2")
|
94
|
-
nabu_jp2 = JP2Writer(fname, psnr=[0])
|
95
|
-
nabu_jp2.write(data)
|
96
|
-
self._check_jp2_file(fname, data)
|
97
|
-
|
98
|
-
def test_2D_lossy(self):
|
99
|
-
fname = path.join(self.tempdir, "sino500_lossy.jp2")
|
100
|
-
nabu_jp2 = JP2Writer(fname, psnr=[80])
|
101
|
-
nabu_jp2.write(self.sino_data)
|
102
|
-
self._check_jp2_file(fname, self.sino_data, expected_psnr=80)
|
103
|
-
|
104
|
-
def test_3D(self):
|
105
|
-
fname = path.join(self.tempdir, "sino500_multi.jp2")
|
106
|
-
n_images = 5
|
107
|
-
data = np.tile(self.sino_data, (n_images, 1, 1))
|
108
|
-
for i in range(n_images):
|
109
|
-
data[i] += i
|
110
|
-
|
111
|
-
nabu_jp2 = JP2Writer(fname, start_index=10)
|
112
|
-
nabu_jp2.write(data)
|
113
|
-
|
114
|
-
assert not (path.isfile(fname))
|
115
|
-
|
116
|
-
prefix, ext = path.splitext(fname)
|
117
|
-
for i in range(data.shape[0]):
|
118
|
-
curr_rel_fname = prefix + str("_%06d" % (i + nabu_jp2.start_index)) + ext
|
119
|
-
curr_fname = path.join(self.tempdir, curr_rel_fname)
|
120
|
-
self._check_jp2_file(curr_fname, data[i])
|
121
|
-
|
122
|
-
|
123
22
|
@pytest.fixture(scope="class")
|
124
23
|
def bootstrap_h5(request):
|
125
24
|
cls = request.cls
|