nabu 2024.1.10__py3-none-any.whl → 2024.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. nabu/__init__.py +1 -1
  2. nabu/app/bootstrap.py +2 -3
  3. nabu/app/cast_volume.py +4 -2
  4. nabu/app/cli_configs.py +5 -0
  5. nabu/app/composite_cor.py +1 -1
  6. nabu/app/create_distortion_map_from_poly.py +5 -6
  7. nabu/app/diag_to_pix.py +7 -19
  8. nabu/app/diag_to_rot.py +14 -29
  9. nabu/app/double_flatfield.py +32 -44
  10. nabu/app/parse_reconstruction_log.py +3 -0
  11. nabu/app/reconstruct.py +53 -15
  12. nabu/app/reconstruct_helical.py +2 -2
  13. nabu/app/stitching.py +27 -13
  14. nabu/app/tests/__init__.py +0 -0
  15. nabu/app/tests/test_reduce_dark_flat.py +4 -1
  16. nabu/cuda/kernel.py +11 -2
  17. nabu/cuda/processing.py +2 -2
  18. nabu/cuda/src/cone.cu +77 -0
  19. nabu/cuda/src/hierarchical_backproj.cu +271 -0
  20. nabu/cuda/utils.py +0 -6
  21. nabu/estimation/alignment.py +5 -19
  22. nabu/estimation/cor.py +173 -599
  23. nabu/estimation/cor_sino.py +356 -26
  24. nabu/estimation/focus.py +63 -11
  25. nabu/estimation/tests/test_cor.py +124 -58
  26. nabu/estimation/tests/test_focus.py +6 -6
  27. nabu/estimation/tilt.py +2 -1
  28. nabu/estimation/utils.py +5 -33
  29. nabu/io/__init__.py +1 -1
  30. nabu/io/cast_volume.py +1 -1
  31. nabu/io/reader.py +416 -21
  32. nabu/io/tests/test_readers.py +422 -0
  33. nabu/io/tests/test_writers.py +1 -102
  34. nabu/io/writer.py +4 -433
  35. nabu/opencl/kernel.py +14 -3
  36. nabu/opencl/processing.py +8 -0
  37. nabu/pipeline/config_validators.py +5 -2
  38. nabu/pipeline/datadump.py +12 -5
  39. nabu/pipeline/estimators.py +162 -188
  40. nabu/pipeline/fullfield/chunked.py +168 -92
  41. nabu/pipeline/fullfield/chunked_cuda.py +7 -3
  42. nabu/pipeline/fullfield/computations.py +2 -7
  43. nabu/pipeline/fullfield/dataset_validator.py +0 -4
  44. nabu/pipeline/fullfield/nabu_config.py +37 -13
  45. nabu/pipeline/fullfield/processconfig.py +22 -13
  46. nabu/pipeline/fullfield/reconstruction.py +13 -9
  47. nabu/pipeline/helical/helical_chunked_regridded.py +1 -1
  48. nabu/pipeline/helical/helical_chunked_regridded_cuda.py +1 -0
  49. nabu/pipeline/helical/helical_reconstruction.py +1 -1
  50. nabu/pipeline/params.py +21 -1
  51. nabu/pipeline/processconfig.py +1 -12
  52. nabu/pipeline/reader.py +146 -0
  53. nabu/pipeline/tests/test_estimators.py +44 -72
  54. nabu/pipeline/utils.py +4 -2
  55. nabu/pipeline/writer.py +10 -2
  56. nabu/preproc/ccd_cuda.py +1 -1
  57. nabu/preproc/ctf.py +14 -7
  58. nabu/preproc/ctf_cuda.py +2 -3
  59. nabu/preproc/double_flatfield.py +5 -12
  60. nabu/preproc/double_flatfield_cuda.py +2 -2
  61. nabu/preproc/flatfield.py +5 -1
  62. nabu/preproc/flatfield_cuda.py +5 -1
  63. nabu/preproc/phase.py +24 -73
  64. nabu/preproc/phase_cuda.py +5 -8
  65. nabu/preproc/tests/test_ctf.py +11 -7
  66. nabu/preproc/tests/test_flatfield.py +67 -122
  67. nabu/preproc/tests/test_paganin.py +54 -30
  68. nabu/processing/azim.py +206 -0
  69. nabu/processing/convolution_cuda.py +1 -1
  70. nabu/processing/fft_cuda.py +15 -17
  71. nabu/processing/histogram.py +2 -0
  72. nabu/processing/histogram_cuda.py +2 -1
  73. nabu/processing/kernel_base.py +3 -0
  74. nabu/processing/muladd_cuda.py +1 -0
  75. nabu/processing/padding_opencl.py +1 -1
  76. nabu/processing/roll_opencl.py +1 -0
  77. nabu/processing/rotation_cuda.py +2 -2
  78. nabu/processing/tests/test_fft.py +17 -10
  79. nabu/processing/unsharp_cuda.py +1 -1
  80. nabu/reconstruction/cone.py +104 -40
  81. nabu/reconstruction/fbp.py +3 -0
  82. nabu/reconstruction/fbp_base.py +7 -2
  83. nabu/reconstruction/filtering.py +20 -7
  84. nabu/reconstruction/filtering_cuda.py +7 -1
  85. nabu/reconstruction/hbp.py +424 -0
  86. nabu/reconstruction/mlem.py +99 -0
  87. nabu/reconstruction/reconstructor.py +2 -0
  88. nabu/reconstruction/rings_cuda.py +19 -19
  89. nabu/reconstruction/sinogram_cuda.py +1 -0
  90. nabu/reconstruction/sinogram_opencl.py +3 -1
  91. nabu/reconstruction/tests/test_cone.py +10 -5
  92. nabu/reconstruction/tests/test_deringer.py +7 -6
  93. nabu/reconstruction/tests/test_fbp.py +124 -10
  94. nabu/reconstruction/tests/test_filtering.py +13 -11
  95. nabu/reconstruction/tests/test_halftomo.py +30 -4
  96. nabu/reconstruction/tests/test_mlem.py +91 -0
  97. nabu/reconstruction/tests/test_reconstructor.py +8 -3
  98. nabu/resources/dataset_analyzer.py +142 -92
  99. nabu/resources/gpu.py +1 -0
  100. nabu/resources/nxflatfield.py +134 -125
  101. nabu/resources/templates/id16a_fluo.conf +42 -0
  102. nabu/resources/tests/test_extract.py +10 -0
  103. nabu/resources/tests/test_nxflatfield.py +2 -2
  104. nabu/stitching/alignment.py +80 -24
  105. nabu/stitching/config.py +105 -68
  106. nabu/stitching/definitions.py +1 -0
  107. nabu/stitching/frame_composition.py +68 -60
  108. nabu/stitching/overlap.py +91 -51
  109. nabu/stitching/single_axis_stitching.py +32 -0
  110. nabu/stitching/slurm_utils.py +6 -6
  111. nabu/stitching/stitcher/__init__.py +0 -0
  112. nabu/stitching/stitcher/base.py +124 -0
  113. nabu/stitching/stitcher/dumper/__init__.py +3 -0
  114. nabu/stitching/stitcher/dumper/base.py +94 -0
  115. nabu/stitching/stitcher/dumper/postprocessing.py +356 -0
  116. nabu/stitching/stitcher/dumper/preprocessing.py +60 -0
  117. nabu/stitching/stitcher/post_processing.py +555 -0
  118. nabu/stitching/stitcher/pre_processing.py +1068 -0
  119. nabu/stitching/stitcher/single_axis.py +484 -0
  120. nabu/stitching/stitcher/stitcher.py +0 -0
  121. nabu/stitching/stitcher/y_stitcher.py +13 -0
  122. nabu/stitching/stitcher/z_stitcher.py +45 -0
  123. nabu/stitching/stitcher_2D.py +278 -0
  124. nabu/stitching/tests/test_config.py +12 -37
  125. nabu/stitching/tests/test_frame_composition.py +33 -59
  126. nabu/stitching/tests/test_overlap.py +149 -7
  127. nabu/stitching/tests/test_utils.py +1 -1
  128. nabu/stitching/tests/test_y_preprocessing_stitching.py +132 -0
  129. nabu/stitching/tests/{test_z_stitching.py → test_z_postprocessing_stitching.py} +167 -561
  130. nabu/stitching/tests/test_z_preprocessing_stitching.py +431 -0
  131. nabu/stitching/utils/__init__.py +1 -0
  132. nabu/stitching/utils/post_processing.py +281 -0
  133. nabu/stitching/utils/tests/test_post-processing.py +21 -0
  134. nabu/stitching/{utils.py → utils/utils.py} +79 -52
  135. nabu/stitching/y_stitching.py +27 -0
  136. nabu/stitching/z_stitching.py +32 -2281
  137. nabu/testutils.py +1 -152
  138. nabu/thirdparty/tomocupy_remove_stripe.py +43 -9
  139. nabu/utils.py +158 -61
  140. {nabu-2024.1.10.dist-info → nabu-2024.2.0.dist-info}/METADATA +24 -17
  141. {nabu-2024.1.10.dist-info → nabu-2024.2.0.dist-info}/RECORD +145 -121
  142. {nabu-2024.1.10.dist-info → nabu-2024.2.0.dist-info}/WHEEL +1 -1
  143. nabu/io/tiffwriter_zmm.py +0 -99
  144. nabu/pipeline/fallback_utils.py +0 -149
  145. nabu/pipeline/helical/tests/test_accumulator.py +0 -158
  146. nabu/pipeline/helical/tests/test_pipeline_elements_full.py +0 -355
  147. nabu/pipeline/helical/tests/test_strategy.py +0 -61
  148. nabu/pipeline/helical/utils.py +0 -51
  149. nabu/pipeline/tests/test_chunk_reader.py +0 -74
  150. {nabu-2024.1.10.dist-info → nabu-2024.2.0.dist-info}/LICENSE +0 -0
  151. {nabu-2024.1.10.dist-info → nabu-2024.2.0.dist-info}/entry_points.txt +0 -0
  152. {nabu-2024.1.10.dist-info → nabu-2024.2.0.dist-info}/top_level.txt +0 -0
@@ -3,12 +3,14 @@ from time import time
3
3
  from math import ceil
4
4
  import numpy as np
5
5
  from silx.io.url import DataUrl
6
- from ...utils import remove_items_from_list
6
+
7
+ from ...utils import get_num_threads, remove_items_from_list
7
8
  from ...resources.logger import LoggerOrPrint
8
9
  from ...resources.utils import extract_parameters
9
- from ...io.reader import ChunkReader, HDF5Loader
10
+ from ...misc.binning import binning as image_binning
11
+ from ...io.reader import EDFStackReader, HDF5Loader, NXTomoReader
10
12
  from ...preproc.ccd import Log, CCDFilter
11
- from ...preproc.flatfield import FlatFieldDataUrls
13
+ from ...preproc.flatfield import FlatField
12
14
  from ...preproc.distortion import DistortionCorrection
13
15
  from ...preproc.shift import VerticalShift
14
16
  from ...preproc.double_flatfield import DoubleFlatField
@@ -16,14 +18,15 @@ from ...preproc.phase import PaganinPhaseRetrieval
16
18
  from ...preproc.ctf import CTFPhaseRetrieval, GeoPars
17
19
  from ...reconstruction.sinogram import SinoNormalization
18
20
  from ...reconstruction.filtering import SinoFilter
21
+ from ...reconstruction.mlem import __have_corrct__, MLEMReconstructor
19
22
  from ...processing.rotation import Rotation
20
23
  from ...reconstruction.rings import MunchDeringer, SinoMeanDeringer, VoDeringer
21
24
  from ...processing.unsharp import UnsharpMask
22
25
  from ...processing.histogram import PartialHistogram, hist_as_2Darray
23
26
  from ..utils import use_options, pipeline_step, get_subregion
27
+ from ..reader import bin_image_stack, load_darks_flats
24
28
  from ..datadump import DataDumpManager
25
29
  from ..writer import WriterManager
26
- from ..detector_distortion_provider import DetectorDistortionProvider
27
30
 
28
31
  # For now we don't have a plain python/numpy backend for reconstruction
29
32
  try:
@@ -41,7 +44,7 @@ class ChunkedPipeline:
41
44
  """
42
45
 
43
46
  backend = "numpy"
44
- FlatFieldClass = FlatFieldDataUrls
47
+ FlatFieldClass = FlatField
45
48
  DoubleFlatFieldClass = DoubleFlatField
46
49
  CCDCorrectionClass = CCDFilter
47
50
  PaganinPhaseRetrievalClass = PaganinPhaseRetrieval
@@ -57,6 +60,8 @@ class ChunkedPipeline:
57
60
  SinoFilterClass = SinoFilter
58
61
  FBPClass = Backprojector
59
62
  ConebeamClass = None # unsupported on CPU
63
+ MLEMClass = MLEMReconstructor
64
+ HBPClass = None # unsupported on CPU
60
65
  HistogramClass = PartialHistogram
61
66
 
62
67
  _default_extra_options = {}
@@ -195,6 +200,7 @@ class ChunkedPipeline:
195
200
  self.logger.debug("Set sub-region to %s" % (str(sub_region)))
196
201
  self.sub_region = sub_region
197
202
  self._sub_region_xz = sub_region[2] + sub_region[1]
203
+ self._radios_were_cropped = False
198
204
 
199
205
  def _set_extra_options(self, extra_options):
200
206
  self.extra_options = self._default_extra_options.copy()
@@ -300,33 +306,72 @@ class ChunkedPipeline:
300
306
  self.datadump_manager._configure_dump("sinogram", force_dump_to_fname=sino_dump_fname)
301
307
  self.logger.debug("Will dump sinogram to %s" % self.datadump_manager.data_dump["sinogram"].fname)
302
308
 
309
+ def _init_reading_processing_function(self):
310
+ # Some processing may be applied directly when reading data (eg. distortion correction, binning, ...)
311
+ # Configure it here
312
+ self._reader_processing_function = None
313
+ self._reader_processing_function_args = None
314
+ self._reader_processing_function_kwargs = None
315
+ self._ff_processing_function = None
316
+ self._ff_processing_function_args = None
317
+ if self.process_config.binning is None or self.process_config.binning == (1, 1):
318
+ return
319
+ if self.dataset_info.kind == "nx":
320
+ self._reader_processing_function = bin_image_stack
321
+ self._reader_processing_function_kwargs = {
322
+ "binning_factor": self.process_config.binning[::-1],
323
+ "num_threads": get_num_threads(),
324
+ }
325
+ else:
326
+ self._reader_processing_function = image_binning
327
+ self._reader_processing_function_args = [self.process_config.binning[::-1]]
328
+ # flat-field is read image-wise
329
+ self._ff_processing_function = image_binning
330
+ self._ff_processing_function_args = [self.process_config.binning[::-1]]
331
+
303
332
  @use_options("read_chunk", "chunk_reader")
304
333
  def _init_reader(self):
305
334
  options = self.processing_options["read_chunk"]
306
- self._update_reader_configuration()
307
-
308
335
  process_file = options.get("process_file", None)
309
- if process_file is None:
310
- # Standard case - start pipeline from raw data
311
- if self.process_config.nabu_config["preproc"]["detector_distortion_correction"] is None:
312
- self.detector_corrector = None
313
- else:
314
- self.detector_corrector = DetectorDistortionProvider(
315
- detector_full_shape_vh=self.process_config.dataset_info.radio_dims[::-1],
316
- correction_type=self.process_config.nabu_config["preproc"]["detector_distortion_correction"],
317
- options=self.process_config.nabu_config["preproc"]["detector_distortion_correction_options"],
318
- )
319
- # ChunkReader always take a non-subsampled dictionary "files".
320
- self.chunk_reader = ChunkReader(
321
- self._read_options["files"],
322
- sub_region=self.sub_region_xz,
323
- data_buffer=self.radios,
324
- pre_allocate=False,
325
- detector_corrector=self.detector_corrector,
326
- convert_float=True,
327
- binning=options["binning"],
328
- dataset_subsampling=options["dataset_subsampling"],
336
+ if process_file is None: # Standard case - start pipeline from raw data
337
+ self._init_reading_processing_function()
338
+
339
+ subs_angles = None
340
+ subs_z = None
341
+ subs_x = None
342
+ if self.process_config.subsampling_factor:
343
+ subs_angles = self.process_config.subsampling_factor
344
+ reader_sub_region = (
345
+ slice(*(self.sub_region[0]) + ((subs_angles,) if subs_angles else ())),
346
+ slice(*(self.sub_region[1]) + ((subs_z,) if subs_z else ())),
347
+ slice(*(self.sub_region[2]) + ((subs_x,) if subs_x else ())),
329
348
  )
349
+
350
+ other_reader_kwargs = {
351
+ "output_dtype": np.float32,
352
+ "processing_func": self._reader_processing_function,
353
+ "processing_func_args": self._reader_processing_function_args,
354
+ "processing_func_kwargs": self._reader_processing_function_kwargs,
355
+ }
356
+
357
+ if self.dataset_info.kind == "nx":
358
+ self.chunk_reader = NXTomoReader(
359
+ self.dataset_info.dataset_hdf5_url.file_path(),
360
+ self.dataset_info.dataset_hdf5_url.data_path(),
361
+ sub_region=reader_sub_region,
362
+ image_key=0,
363
+ **other_reader_kwargs,
364
+ )
365
+ elif self.dataset_info.kind == "edf":
366
+ files = [
367
+ self.dataset_info.projections[k].file_path() for k in sorted(self.dataset_info.projections.keys())
368
+ ]
369
+ self.chunk_reader = EDFStackReader(
370
+ files,
371
+ sub_region=reader_sub_region,
372
+ n_reading_threads=max(1, get_num_threads() // 2),
373
+ **other_reader_kwargs,
374
+ )
330
375
  else:
331
376
  # Resume pipeline from dumped intermediate step
332
377
  self.chunk_reader = HDF5Loader(
@@ -341,36 +386,16 @@ class ChunkedPipeline:
341
386
  "Load subregion %s from file %s" % (str(self.chunk_reader.sub_region), self.chunk_reader.fname)
342
387
  )
343
388
 
344
- def _update_reader_configuration(self):
345
- """
346
- Modify self.processing_options["read_chunk"] to select a subset of the files, if needed
347
- (i.e when processing only a subset of the images stack)
348
- """
349
- self._read_options = self.processing_options["read_chunk"].copy()
350
- if self.n_angles == self.process_config.n_angles(subsampling=True):
351
- # Nothing to do if the full angular range is processed in one shot
352
- return
353
- if self._resume_from_step is not None:
354
- if self._resume_from_step == "sinogram":
355
- msg = "It makes no sense to use 'grouped processing' when resuming from sinogram"
356
- self.logger.fatal(msg)
357
- raise ValueError(msg)
358
- # Nothing to do if we resume the processing from a given step
359
- return
360
- input_data_files = {}
361
- files_indices = sorted(self._read_options["files"].keys())
362
- angle_idx_start, angle_idx_end = self.sub_region[0]
363
- for i in range(angle_idx_start, angle_idx_end):
364
- idx = files_indices[i]
365
- input_data_files[idx] = self._read_options["files"][idx]
366
- self._read_options["files"] = input_data_files
367
-
368
389
  @use_options("flatfield", "flatfield")
369
390
  def _init_flatfield(self):
370
391
  self._ff_options = self.processing_options["flatfield"].copy()
371
- # Use chunk_reader.files instead of process_config.projs_indices(subsampling=True), because
372
- # chunk_reader might read only a subset of the files (in "grouped mode")
373
- self._ff_options["projs_indices"] = list(self.chunk_reader.files_subsampled.keys())
392
+
393
+ # This won't work when resuming from a step (i.e before FF), because we rely on H5Loader()
394
+ # which re-compacts the data. When data is re-compacted, we have to know the original radios positions.
395
+ # These positions can be saved in the "file_dump" metadata, but it is not loaded for now
396
+ # (the process_config object is re-built from scratch every time)
397
+ self._ff_options["projs_indices"] = self.chunk_reader.get_frames_indices()
398
+
374
399
  if self._ff_options.get("normalize_srcurrent", False):
375
400
  a_start_idx, a_end_idx = self.sub_region[0]
376
401
  subs = self.process_config.subsampling_factor
@@ -379,7 +404,7 @@ class ChunkedPipeline:
379
404
  distortion_correction = None
380
405
  if self._ff_options["do_flat_distortion"]:
381
406
  self.logger.info("Flats distortion correction will be applied")
382
- self.FlatFieldClass = FlatFieldDataUrls # no GPU implementation available, force this backend
407
+ self.FlatFieldClass = FlatField # no GPU implementation available, force this backend
383
408
  estimation_kwargs = {}
384
409
  estimation_kwargs.update(self._ff_options["flat_distortion_params"])
385
410
  estimation_kwargs["logger"] = self.logger
@@ -387,20 +412,25 @@ class ChunkedPipeline:
387
412
  estimation_method="fft-correlation", estimation_kwargs=estimation_kwargs, correction_method="interpn"
388
413
  )
389
414
 
415
+ # Reduced darks/flats are loaded, but we have to crop them on the current sub-region
416
+ # and possibly do apply some pre-processing (binning, distortion correction, ...)
417
+ darks_flats = load_darks_flats(
418
+ self.dataset_info,
419
+ self.sub_region[1:],
420
+ processing_func=self._ff_processing_function,
421
+ processing_func_args=self._ff_processing_function_args,
422
+ )
423
+
390
424
  # FlatField parameter "radios_indices" must account for subsampling
391
425
  self.flatfield = self.FlatFieldClass(
392
426
  self.radios_shape,
393
- flats=self.dataset_info.flats,
394
- darks=self.dataset_info.darks,
427
+ flats=darks_flats["flats"],
428
+ darks=darks_flats["darks"],
395
429
  radios_indices=self._ff_options["projs_indices"],
396
430
  interpolation="linear",
397
431
  distortion_correction=distortion_correction,
398
- sub_region=self.sub_region_xz,
399
- detector_corrector=self.detector_corrector,
400
- binning=self._ff_options["binning"],
401
432
  radios_srcurrent=self._ff_options["radios_srcurrent"],
402
433
  flats_srcurrent=self._ff_options["flats_srcurrent"],
403
- convert_float=True,
404
434
  )
405
435
 
406
436
  @use_options("double_flatfield", "double_flatfield")
@@ -421,8 +451,7 @@ class ChunkedPipeline:
421
451
  self.double_flatfield = self.DoubleFlatFieldClass(
422
452
  self.radios_shape,
423
453
  result_url=result_url,
424
- sub_region=self.sub_region_xz,
425
- detector_corrector=self.detector_corrector,
454
+ sub_region=self.sub_region[1:],
426
455
  input_is_mlog=False,
427
456
  output_is_mlog=False,
428
457
  average_is_on_log=avg_is_on_log,
@@ -438,13 +467,15 @@ class ChunkedPipeline:
438
467
  self.radios_shape[1:], median_clip_thresh=options["median_clip_thresh"]
439
468
  )
440
469
 
441
- @use_options("rotate_projections", "projs_rot")
470
+ @use_options("tilt_correction", "projs_rot")
442
471
  def _init_radios_rotation(self):
443
- options = self.processing_options["rotate_projections"]
472
+ options = self.processing_options["tilt_correction"]
444
473
  center = options["center"]
445
474
  if center is None:
446
- nx, ny = self.radios_shape[1:][::-1] # after binning
447
- center = (nx / 2 - 0.5, ny / 2 - 0.5)
475
+ nz, nx = self.radios_shape[1:] # after binning
476
+ center_x = self.process_config.rotation_axis_position(binning=True)
477
+ center_z = nz / 2 - 0.5
478
+ center = (center_x, center_z)
448
479
  center = (center[0], center[1] - self.z_min)
449
480
  self.projs_rot = self.ImageRotationClass(
450
481
  self.radios_shape[1:], options["angle"], center=center, mode="edge", reshape=False
@@ -545,10 +576,13 @@ class ChunkedPipeline:
545
576
  raise ValueError("No usable FBP module was found")
546
577
  if options["method"] == "cone" and self.ConebeamClass is None:
547
578
  raise ValueError("No usable cone-beam module was found")
579
+ if options["method"] == "mlem" and self.MLEMClass is None:
580
+ raise ValueError("No usable MLEM module was found.")
548
581
 
549
- if options["method"] == "FBP":
550
- n_slices = self.n_slices
551
- self.reconstruction = self.FBPClass(
582
+ n_slices = self.n_slices
583
+ if options["method"] in ["FBP", "HBP"]: # both have the same API
584
+ rec_cls = self.HBPClass if options["method"] == "HBP" else self.FBPClass
585
+ self.reconstruction = rec_cls(
552
586
  self.sinos_shape[1:],
553
587
  angles=options["angles"],
554
588
  rot_center=options["rotation_axis_position"],
@@ -561,7 +595,10 @@ class ChunkedPipeline:
561
595
  "axis_correction": options["axis_correction"],
562
596
  "centered_axis": options["centered_axis"],
563
597
  "clip_outer_circle": options["clip_outer_circle"],
598
+ "outer_circle_value": options["outer_circle_value"],
564
599
  "filter_cutoff": options["fbp_filter_cutoff"],
600
+ "hbp_legs": options["hbp_legs"],
601
+ "hbp_reduction_steps": options["hbp_reduction_steps"],
565
602
  },
566
603
  )
567
604
 
@@ -577,11 +614,33 @@ class ChunkedPipeline:
577
614
  sample_detector_dist,
578
615
  angles=-options["angles"],
579
616
  rot_center=options["rotation_axis_position"],
580
- axis_correction=(-options["axis_correction"] if options["axis_correction"] is not None else None),
581
617
  pixel_size=1,
582
- scale_factor=1.0 / options["voxel_size_cm"][0],
583
618
  padding_mode=options["padding_type"],
584
619
  slice_roi=self.process_config.rec_roi,
620
+ extra_options={
621
+ "scale_factor": 1.0 / options["voxel_size_cm"][0],
622
+ "axis_correction": -options["axis_correction"] if options["axis_correction"] is not None else None,
623
+ "clip_outer_circle": options["clip_outer_circle"],
624
+ "outer_circle_value": options["outer_circle_value"],
625
+ "filter_cutoff": options["fbp_filter_cutoff"],
626
+ },
627
+ )
628
+
629
+ if options["method"] == "mlem" and options["implementation"] in (None, "corrct"):
630
+ self.reconstruction = self.MLEMClass( # pylint: disable=E1102
631
+ (self.radios_shape[1],) + self.sino_shape,
632
+ angles_rad=-options["angles"], # WARNING: mind the sign...
633
+ shifts_uv=self.dataset_info.translations, # In config file, one line per proj, each line is (tu,tv). Corrct expects one col per proj and (tv,tu).
634
+ cor=options["rotation_axis_position"],
635
+ n_iterations=options["iterations"],
636
+ extra_options={
637
+ "compute_shifts": False,
638
+ "tomo_consistency": False,
639
+ "v_min_for_v_shifts": 0,
640
+ "v_max_for_v_shifts": None,
641
+ "v_min_for_u_shifts": 0,
642
+ "v_max_for_u_shifts": None,
643
+ },
585
644
  )
586
645
 
587
646
  self._allocate_recs(*self.process_config.rec_shape, n_slices=n_slices)
@@ -609,7 +668,10 @@ class ChunkedPipeline:
609
668
  "jpeg2000_compression_ratio": options["jpeg2000_compression_ratio"],
610
669
  "float_clip_values": options["float_clip_values"],
611
670
  "tiff_single_file": options.get("tiff_single_file", False),
612
- "single_output_file_initialized": getattr(self.process_config, "single_output_file_initialized", False),
671
+ "single_output_file_initialized": getattr(
672
+ self.process_config, "single_output_file_initialized", False
673
+ ), # COMPAT.
674
+ "writer_initialized": getattr(self.process_config, "_writer_initialized", False),
613
675
  "raw_vol_metadata": {"voxelSize": self.dataset_info.pixel_size}, # legacy...
614
676
  }
615
677
  writer_extra_options.update(extra_options)
@@ -633,10 +695,9 @@ class ChunkedPipeline:
633
695
  def _read_data(self):
634
696
  self.logger.debug("Region = %s" % str(self.sub_region))
635
697
  t0 = time()
636
- self.chunk_reader.load_data()
698
+ self.chunk_reader.load_data(output=self.radios)
637
699
  el = time() - t0
638
- shp = self.chunk_reader.data.shape
639
- self.logger.info("Read subvolume %s in %.2f s" % (str(shp), el))
700
+ self.logger.info("Read subvolume %s in %.2f s" % (str(self.radios.shape), el))
640
701
 
641
702
  @pipeline_step("flatfield", "Applying flat-field")
642
703
  def _flatfield(self):
@@ -689,7 +750,7 @@ class ChunkedPipeline:
689
750
  def _crop_radios(self):
690
751
  if self.use_margin:
691
752
  self._orig_radios = self.radios
692
- if self.processing_options.get("reconstruction", {}).get("method", None) == "cone":
753
+ if self.processing_options.get("reconstruction", {}).get("method", None) in ("cone",):
693
754
  return
694
755
  ((U, D), (L, R)) = self.margin
695
756
  self.logger.debug(
@@ -697,6 +758,7 @@ class ChunkedPipeline:
697
758
  )
698
759
  U, D, L, R = U or None, -D or None, L or None, -R or None
699
760
  self.radios = self.radios[:, U:D, L:R] # view
761
+ self._radios_were_cropped = True
700
762
 
701
763
  @pipeline_step("sino_normalization", "Normalizing sinograms")
702
764
  def _normalize_sinos(self, radios=None):
@@ -725,6 +787,10 @@ class ChunkedPipeline:
725
787
  self._reconstruct_cone()
726
788
  return
727
789
 
790
+ if options["method"] == "mlem":
791
+ self.recs = self._reconstruct_mlem()
792
+ return
793
+
728
794
  for i in range(self.n_slices):
729
795
  self._tmp_sino[:] = self.radios[:, i, :] # copy into contiguous array
730
796
  self.reconstruction.fbp(self._tmp_sino, output=self.recs[i])
@@ -754,6 +820,26 @@ class ChunkedPipeline:
754
820
  relative_z_position=((z_min + z_max) / self.process_config.binning_z / 2) - n_z_tot / 2,
755
821
  )
756
822
 
823
+ def _reconstruct_mlem(self):
824
+ """
825
+ This reconstructs the entire sinograms stack at once
826
+ """
827
+
828
+ n_angles, n_z, n_x = self.radios.shape
829
+
830
+ # FIXME
831
+ # can't do a discontiguous single copy...
832
+ # Initially done for Astra CB recons. But happens that MLEM Corrct also expects
833
+ # data with this order (nb_rows, nb_angles, nb_cols)
834
+ data_vwu = self._allocate_array((n_z, n_angles, n_x), np.float32, "sinos_mlem")
835
+ for i in range(n_z):
836
+ data_vwu[i] = self.radios[:, i, :]
837
+ # ---
838
+
839
+ return self.reconstruction.reconstruct( # pylint: disable=E1101
840
+ data_vwu,
841
+ )
842
+
757
843
  @pipeline_step("histogram", "Computing histogram")
758
844
  def _compute_histogram(self, data=None):
759
845
  if data is None:
@@ -770,7 +856,8 @@ class ChunkedPipeline:
770
856
  self.writer.write_data(data)
771
857
  self.logger.info("Wrote %s" % self.writer.fname)
772
858
  self._write_histogram()
773
- self.process_config.single_output_file_initialized = True
859
+ self.process_config.single_output_file_initialized = True # COMPAT.
860
+ self.process_config._writer_initialized = True
774
861
 
775
862
  def _write_histogram(self):
776
863
  if "histogram" not in self.processing_steps:
@@ -829,24 +916,13 @@ class ChunkedPipeline:
829
916
  self._process_finalize()
830
917
 
831
918
  def _reset_reader_subregion(self):
832
- if self._resume_from_step is None:
833
- # Normal mode - read data from raw radios
834
- self.chunk_reader._set_subregion(self.sub_region_xz)
835
- self.chunk_reader._init_reader()
836
- self.chunk_reader._loaded = False
837
- else:
838
- # Resume from a checkpoint. In this case, we have to re-initialize "datadump manager"
839
- # sooner to configure start_xyz, end_xyz
840
- self._init_data_dump()
919
+ if self._resume_from_step is not None:
841
920
  self.chunk_reader._set_subregion(self.datadump_manager.get_read_dump_subregion())
842
- self.chunk_reader._loaded = False
843
- if self._grouped_processing:
844
- self._update_reader_configuration()
845
- self.chunk_reader._set_files(self._read_options["files"])
921
+ self._init_data_dump()
922
+ self._init_reader()
846
923
 
847
924
  def _reset_sub_region(self, sub_region):
848
925
  self.set_subregion(sub_region)
849
- # When sub_region is changed, all components involving files reading have to be updated
850
926
  self._reset_reader_subregion()
851
927
  self._init_flatfield() # reset flatfield
852
928
  self._init_writer()
@@ -1,5 +1,5 @@
1
1
  from ...preproc.ccd_cuda import CudaLog, CudaCCDFilter
2
- from ...preproc.flatfield_cuda import CudaFlatFieldDataUrls
2
+ from ...preproc.flatfield_cuda import CudaFlatField
3
3
  from ...preproc.shift_cuda import CudaVerticalShift
4
4
  from ...preproc.double_flatfield_cuda import CudaDoubleFlatField
5
5
  from ...preproc.phase_cuda import CudaPaganinPhaseRetrieval
@@ -11,6 +11,7 @@ from ...processing.unsharp_cuda import CudaUnsharpMask
11
11
  from ...processing.rotation_cuda import CudaRotation
12
12
  from ...processing.histogram_cuda import CudaPartialHistogram
13
13
  from ...reconstruction.fbp import Backprojector
14
+ from ...reconstruction.hbp import HierarchicalBackprojector
14
15
  from ...reconstruction.cone import __have_astra__, ConebeamReconstructor
15
16
  from ...cuda.utils import get_cuda_context, __has_pycuda__, __pycuda_error_msg__
16
17
  from ..utils import pipeline_step
@@ -28,7 +29,7 @@ class CudaChunkedPipeline(ChunkedPipeline):
28
29
  """
29
30
 
30
31
  backend = "cuda"
31
- FlatFieldClass = CudaFlatFieldDataUrls
32
+ FlatFieldClass = CudaFlatField
32
33
  DoubleFlatFieldClass = CudaDoubleFlatField
33
34
  CCDCorrectionClass = CudaCCDFilter
34
35
  PaganinPhaseRetrievalClass = CudaPaganinPhaseRetrieval
@@ -45,6 +46,7 @@ class CudaChunkedPipeline(ChunkedPipeline):
45
46
  SinoFilterClass = CudaSinoFilter
46
47
  FBPClass = Backprojector
47
48
  ConebeamClass = ConebeamReconstructor
49
+ HBPClass = HierarchicalBackprojector
48
50
  HistogramClass = CudaPartialHistogram
49
51
 
50
52
  def __init__(
@@ -91,7 +93,7 @@ class CudaChunkedPipeline(ChunkedPipeline):
91
93
  d_arr = getattr(self, d_name, None)
92
94
  if d_arr is None:
93
95
  self.logger.debug("Allocating %s: %s" % (name, str(shape)))
94
- d_arr = garray.zeros(shape, dtype)
96
+ d_arr = garray.zeros(shape, dtype) # pylint: disable=E0606
95
97
  setattr(self, d_name, d_arr)
96
98
  return d_arr
97
99
 
@@ -127,6 +129,8 @@ class CudaChunkedPipeline(ChunkedPipeline):
127
129
  U, D = U or None, -D or None
128
130
  # not sure why slicing can't be done before get()
129
131
  self.recs = self.recs.get()[U:D, ...]
132
+ elif self.processing_options["reconstruction"]["method"] == "mlem":
133
+ pass
130
134
  else:
131
135
  self.recs = self.recs.get()
132
136
 
@@ -128,13 +128,7 @@ def estimate_required_memory(
128
128
  if process_config.rec_params["method"] == "cone":
129
129
  # In cone-beam reconstruction, need both sinograms and reconstruction inside GPU.
130
130
  # That's big!
131
- # Even more if padding is done at the nabu side.
132
- if rec_config["padding_type"] == "zeros":
133
- total_memory_needed += 2 * data_volume_size
134
- else:
135
- total_memory_needed += (
136
- data_volume_size + get_next_power(2 * Nx) * Na * Nz * 4 * 4
137
- ) # no idea why the last *4 is necessary
131
+ total_memory_needed += 2 * data_volume_size
138
132
 
139
133
  if debug:
140
134
  print(
@@ -216,6 +210,7 @@ def estimate_max_chunk_size(
216
210
  delta_z = 0
217
211
 
218
212
  mem = 0
213
+ # pylint: disable=E0606, E0601
219
214
  last_valid_delta_a = delta_a
220
215
  last_valid_delta_z = delta_z
221
216
  while True:
@@ -19,12 +19,8 @@ class FullFieldDatasetValidator(DatasetValidatorBase):
19
19
  if self.nabu_config["preproc"]["flatfield"]:
20
20
  darks = self.dataset_info.darks
21
21
  assert len(darks) > 0, "Need at least one dark to perform flat-field correction"
22
- for dark_id, dark_url in darks.items():
23
- assert os.path.isfile(dark_url.file_path()), "Dark file %s not found" % dark_url.file_path()
24
22
  flats = self.dataset_info.flats
25
23
  assert len(flats) > 0, "Need at least one flat to perform flat-field correction"
26
- for flat_id, flat_url in flats.items():
27
- assert os.path.isfile(flat_url.file_path()), "Flat file %s not found" % flat_url.file_path()
28
24
 
29
25
  def _check_slice_indices(self):
30
26
  nx, nz = self.dataset_info.radio_dims
@@ -10,7 +10,7 @@ nabu_config = {
10
10
  },
11
11
  "hdf5_entry": {
12
12
  "default": "",
13
- "help": "Entry in the HDF5 file, if applicable. Default is the first available entry.",
13
+ "help": "Which entry to process in the data HDF5 file. Default is the first entry. It can be a comma-separated list of entries, and/or a wildcard (* for all entries, or things like entry???1).",
14
14
  "validator": optional_string_validator,
15
15
  "type": "advanced",
16
16
  },
@@ -169,15 +169,9 @@ nabu_config = {
169
169
  "validator": generic_options_validator,
170
170
  "type": "advanced",
171
171
  },
172
- "rotate_projections": {
173
- "default": "",
174
- "help": "Whether to rotate each projection image with a certain angle (in degree). By default (empty) no rotation is done.",
175
- "validator": optional_nonzero_float_validator,
176
- "type": "advanced",
177
- },
178
172
  "rotate_projections_center": {
179
173
  "default": "",
180
- "help": "Center of rotation when 'rotate_projections' is non-empty. By default the center of rotation is the middle of each radio, i.e ((Nx-1)/2.0, (Ny-1)/2.0).",
174
+ "help": "Center of rotation when 'tilt_correction' is non-empty. By default the center of rotation is the middle of each radio, i.e ((Nx-1)/2.0, (Ny-1)/2.0).",
181
175
  "validator": optional_tuple_of_floats_validator,
182
176
  "type": "advanced",
183
177
  },
@@ -247,10 +241,16 @@ nabu_config = {
247
241
  "reconstruction": {
248
242
  "method": {
249
243
  "default": "FBP",
250
- "help": "Reconstruction method. Possible values: FBP, cone, none. If value is 'none', no reconstruction will be done.",
244
+ "help": "Reconstruction method. Possible values: FBP, HBP, cone, MLEM, none. If value is 'none', no reconstruction will be done.",
251
245
  "validator": reconstruction_method_validator,
252
246
  "type": "required",
253
247
  },
248
+ "implementation": {
249
+ "default": "",
250
+ "help": "Reconstruction method implementation. The same method can have several implementations. Can be 'nabu', 'corrct', 'astra'",
251
+ "validator": reconstruction_implementation_validator,
252
+ "type": "advanced",
253
+ },
254
254
  "angles_file": {
255
255
  "default": "",
256
256
  "help": "In the case you want to override the angles found in the files metadata. The angles are in degree.",
@@ -259,7 +259,7 @@ nabu_config = {
259
259
  },
260
260
  "rotation_axis_position": {
261
261
  "default": "sliding-window",
262
- "help": "Rotation axis position. It can be a number or the name of an estimation method (empty value means the middle of the detector).\nThe following methods are available to find automatically the Center of Rotation (CoR):\n - centered : a fast and simple auto-CoR method. It only works when the CoR is not far from the middle of the detector. It does not work for half-tomography.\n - global : a slow but robust auto-CoR.\n - sliding-window : semi-automatically find the CoR with a sliding window. You have to specify on which side the CoR is (left, center, right). Please see the 'cor_options' parameter.\n - growing-window : automatically find the CoR with a sliding-and-growing window. You can tune the option with the parameter 'cor_options'.\n - sino-coarse-to-fine: Estimate CoR from sinogram. Only works for 360 degrees scans.\n - composite-coarse-to-fine: Estimate CoR from composite multi-angle images. Only works for 360 degrees scans.\n - fourier-angles: Estimate CoR from sino based on an angular correlation analysis. You can tune the option with the parameter 'cor_options'.\n - octave-accurate: Legacy from octave accurate COR estimation algorithm. It first estimates the COR with global fourier-based correlation, then refines this estimation with local correlation based on the variance of the difference patches. You can tune the option with the parameter 'cor_options'.",
262
+ "help": "Rotation axis position. It can be a number or the name of an estimation method (empty value means the middle of the detector).\nThe following methods are available to find automatically the Center of Rotation (CoR):\n - centered : a fast and simple auto-CoR method. It only works when the CoR is not far from the middle of the detector. It does not work for half-tomography.\n - global : a slow but robust auto-CoR.\n - sliding-window : semi-automatically find the CoR with a sliding window. You have to specify on which side the CoR is (left, center, right). Please see the 'cor_options' parameter.\n - growing-window : automatically find the CoR with a sliding-and-growing window. You can tune the option with the parameter 'cor_options'.\n - sino-coarse-to-fine: Estimate CoR from sinogram. Only works for 360 degrees scans.\n - composite-coarse-to-fine: Estimate CoR from composite multi-angle images. Only works for 360 degrees scans.\n - fourier-angles: Estimate CoR from sino based on an angular correlation analysis. You can tune the option with the parameter 'cor_options'.\n - octave-accurate: Legacy from octave accurate COR estimation algorithm. It first estimates the COR with global fourier-based correlation, then refines this estimation with local correlation based on the variance of the difference patches. You can tune the option with the parameter 'cor_options'.\n - vo: Method from Nghia Vo, based on double-wedge in sinogram Fourier transform (needs algotom python package)",
263
263
  "validator": cor_validator,
264
264
  "type": "required",
265
265
  },
@@ -283,7 +283,7 @@ nabu_config = {
283
283
  },
284
284
  "translation_movements_file": {
285
285
  "default": "",
286
- "help": "A file where each line describes the horizontal and vertical translations of the sample (or detector). The order is 'horizontal, vertical'.",
286
+ "help": "A file where each line describes the horizontal and vertical translations of the sample (or detector). The order is 'horizontal, vertical'.\nIt can be created from a numpy array saved with 'numpy.savetxt'",
287
287
  "validator": optional_values_file_validator,
288
288
  "type": "advanced",
289
289
  },
@@ -331,16 +331,34 @@ nabu_config = {
331
331
  },
332
332
  "clip_outer_circle": {
333
333
  "default": "0",
334
- "help": "Whether to set to zero voxels falling outside of the reconstruction region",
334
+ "help": "Whether to mask voxels falling outside of the reconstruction region",
335
335
  "validator": boolean_validator,
336
336
  "type": "optional",
337
337
  },
338
+ "outer_circle_value": {
339
+ "default": "0",
340
+ "help": "If 'clip_outer_circle' is enabled, value of the voxels falling outside of the reconstruction region.",
341
+ "validator": float_validator,
342
+ "type": "optional",
343
+ },
338
344
  "centered_axis": {
339
345
  "default": "1",
340
346
  "help": "If set to true, the reconstructed region is centered on the rotation axis, i.e the center of the image will be the rotation axis position.",
341
347
  "validator": boolean_validator,
342
348
  "type": "optional",
343
349
  },
350
+ "hbp_reduction_steps": {
351
+ "default": "2",
352
+ "help": "How many reduction steps will be taken. At least 2. A Higher number may increase speed but may also increase the interpolation errors",
353
+ "validator": nonnegative_integer_validator,
354
+ "type": "advanced",
355
+ },
356
+ "hbp_legs": {
357
+ "default": "4",
358
+ "help": "Increasing this parameter help matching the GPU memory size for big slices. Reconstruction by fragments of the whole images. For very large slices it can be useful to increase this number to fit the memory",
359
+ "validator": nonnegative_integer_validator,
360
+ "type": "advanced",
361
+ },
344
362
  "start_x": {
345
363
  "default": "0",
346
364
  "help": "\nParameters for sub-volume reconstruction. Indices start at 0, and upper bounds are INCLUDED!\n----------------------------------------------------------------\n(x, y) are the dimension of a slice, and (z) is the 'vertical' axis\nBy default, all the volume is reconstructed slice by slice, along the axis 'z'.",
@@ -381,7 +399,7 @@ nabu_config = {
381
399
  "default": "200",
382
400
  "help": "\nParameters for iterative algorithms\n------------------------------------\nNumber of iterations",
383
401
  "validator": nonnegative_integer_validator,
384
- "type": "unsupported",
402
+ "type": "advanced",
385
403
  },
386
404
  "optim_algorithm": {
387
405
  "default": "chambolle-pock",
@@ -587,4 +605,10 @@ renamed_keys = {
587
605
  "since": "2021.2.0",
588
606
  "message": "Option 'flatfield_enabled' has been renamed 'flatfield' in [preproc]",
589
607
  },
608
+ "rotate_projections": {
609
+ "section": "preproc",
610
+ "new_name": "",
611
+ "since": "2024.2.0",
612
+ "message": "Option 'rotate_projections' removed as it was duplicate of 'tilt_correction'. Please use the latter with a scalar value.",
613
+ },
590
614
  }