nabu 2022.3.0a1__py3-none-any.whl → 2023.1.0a2__py3-none-any.whl

nabu/pipeline/estimators.py

@@ -4,7 +4,7 @@ nabu.pipeline.estimators: helper classes/functions to estimate parameters of a d
 """
 import inspect
 import numpy as np
-import scipy.fft
+import scipy.fft  #  pylint: disable=E0611
 from silx.io import get_data
 from ..preproc.flatfield import FlatFieldDataUrls
 from ..estimation.cor import (
@@ -26,6 +26,7 @@ from ..io.reader import ChunkReader
 from ..preproc.ccd import Log, CCDFilter
 from ..misc import fourier_filters
 from .params import cor_methods
+from ..io.reader import load_images_from_dataurl_dict
 
 
 def estimate_cor(method, dataset_info, do_flatfield=True, cor_options_str=None, logger=None):
@@ -97,15 +98,17 @@ class CORFinderBase:
 
     def _init_cor_finder(self, method, cor_options):
         self.method = method
-        self.cor_options = cor_options
+        if not isinstance(cor_options, (type(None), dict)):
+            raise TypeError(
+                f"cor_options is expected to be an optional instance of dict. Get {cor_options} ({type(cor_options)}) instead"
+            )
+        self.cor_options = cor_options or {}
 
         cor_class = self.search_methods[method]["class"]
         self.cor_finder = cor_class(logger=self.logger)
 
-        cor_options = cor_options or {}
-        lookup_side = cor_options.get("side", "center")
-        if self.dataset_info.is_halftomo:
-            lookup_side = "right"
+        default_lookup_side = "right" if self.dataset_info.is_halftomo else "center"
+        lookup_side = self.cor_options.get("side", default_lookup_side)
 
         self.cor_exec_args = []
         self.cor_exec_args.extend(self.search_methods[method].get("default_args", []))
@@ -366,7 +369,6 @@ class CompositeCORFinder:
         spike_threshold=0.04,
         logger=None,
     ):
-
         self.dataset_info = dataset_info
         self.logger = LoggerOrPrint(logger)
 
@@ -432,15 +434,22 @@ class CompositeCORFinder:
 
         self.absolute_indices = sorted(self.dataset_info.projections.keys())
 
-        self.flatfield = FlatFieldDataUrls(
-            (len(self.absolute_indices), self.sy, self.sx),
-            self.dataset_info.flats,
-            self.dataset_info.darks,
-            radios_indices=self.absolute_indices,
-            dtype=np.float64,
-        )
-        self.mlog = Log((1,) + self.flatfield.shape, clip_min=1e-6, clip_max=10.0)
+        my_flats = load_images_from_dataurl_dict(self.dataset_info.flats)
+
+        if my_flats is not None and len(list(my_flats.keys())):
+            self.use_flat = True
+            self.flatfield = FlatFieldDataUrls(
+                (len(self.absolute_indices), self.sy, self.sx),
+                self.dataset_info.flats,
+                self.dataset_info.darks,
+                radios_indices=self.absolute_indices,
+                dtype=np.float64,
+            )
+        else:
+            self.use_flat = False
+
         self.sx, self.sy = self.dataset_info.radio_dims
+        self.mlog = Log((1,) + (self.sy, self.sx), clip_min=1e-6, clip_max=10.0)
         self.rcor_abs = round(self.sx / 2.0)
         self.cor_acc = round(self.sx / 2.0)
 
@@ -483,9 +492,15 @@ class CompositeCORFinder:
         if self.dataset_info.is_halftomo:
             default_dict["side"] = "right"
 
-        if cor_options is None or cor_options is "":
+        if cor_options is None or cor_options == "":
             cor_options = {}
-
+        if isinstance(cor_options, str):
+            try:
+                cor_options = extract_parameters(cor_options, sep=";")
+            except Exception as exc:
+                msg = "Could not extract parameters from cor_options: %s" % (str(exc))
+                self.logger.fatal(msg)
+                raise ValueError(msg)
         default_dict.update(cor_options)
         cor_options = default_dict
 
@@ -496,7 +511,8 @@ class CompositeCORFinder:
         radio_dataset_idx = image_num
         data_url = self.dataset_info.projections[radio_dataset_idx]
         radio = get_data(data_url).astype(np.float64)
-        self.flatfield.normalize_single_radio(radio, radio_dataset_idx, dtype=radio.dtype)
+        if self.use_flat:
+            self.flatfield.normalize_single_radio(radio, radio_dataset_idx, dtype=radio.dtype)
         if self.take_log:
             self.mlog.take_logarithm(radio)
 
@@ -588,7 +604,6 @@ class CompositeCORFinder:
             overlap_max = min(2 * ovsd_sx - 4, 2 * ovsd_sx - self.ovs * self.ovs * self.high_pass * 3)
 
         elif self.cor_options["side"] == "near":
-
             near_pos = self.cor_options["near_pos"]
             near_width = self.cor_options["near_width"]
 
@@ -644,12 +659,11 @@ class CompositeCORFinder:
                 best_value = min_value
             self.logger.debug(
                 "testing an overlap of %.2f pixels, actual best overlap is %.2f pixels over %d\r"
-                % (z / self.ovs, best_overlap / self.ovs, ovsd_sx),
+                % (z / self.ovs, best_overlap / self.ovs, ovsd_sx / self.ovs),
             )
 
         offset = (ovsd_sx - best_overlap) / self.ovs / 2
         cor_abs = (self.sx - 1) / 2 + offset
-        self.logger.info("Found an optimal cor at %e (absolute)\n" % cor_abs)
 
         return cor_abs
 

nabu/pipeline/fallback_utils.py

@@ -129,7 +129,7 @@ class WriterConfigurator:
                 raise ValueError(err)
 
     def _init_separate_histogram_writer(self, hist_entry):
-        hist_fname = path.join(self.output_dir, "histogram_%04d.hdf5" % self.start_index)
+        hist_fname = path.join(self.output_dir, "histogram_%06d.hdf5" % self.start_index)
         self.histogram_writer = LegacyNXProcessWriter(
             hist_fname,
             entry=hist_entry,

nabu/pipeline/fullfield/chunked.py

@@ -22,6 +22,7 @@ from ...misc.histogram import PartialHistogram, hist_as_2Darray
 from ..utils import use_options, pipeline_step, get_subregion
 from ..datadump import DataDumpManager
 from ..writer import WriterManager
+from ..detector_distortion_provider import DetectorDistortionProvider
 
 # For now we don't have a plain python/numpy backend for reconstruction
 try:
@@ -61,7 +62,9 @@ class ChunkedPipeline:
     # The first stage will skip these steps, and the second stage will do these stages after merging sinograms.
     _reconstruction_steps = ["sino_rings_correction", "reconstruction", "save", "histogram"]
 
-    def __init__(self, process_config, chunk_shape, margin=None, logger=None, extra_options=None):
+    def __init__(
+        self, process_config, chunk_shape, margin=None, logger=None, use_grouped_mode=False, extra_options=None
+    ):
         """
         Initialize a "Chunked" pipeline.
 
@@ -96,15 +99,15 @@ class ChunkedPipeline:
         of the volume).
         """
         self.logger = LoggerOrPrint(logger)
-        self._set_params(process_config, chunk_shape, extra_options, margin)
+        self._set_params(process_config, chunk_shape, extra_options, margin, use_grouped_mode)
         self._init_pipeline()
 
-    def _set_params(self, process_config, chunk_shape, extra_options, margin):
+    def _set_params(self, process_config, chunk_shape, extra_options, margin, use_grouped_mode):
         self.process_config = process_config
         self.dataset_info = self.process_config.dataset_info
         self.processing_steps = self.process_config.processing_steps.copy()
         self.processing_options = self.process_config.processing_options
-        self._set_chunk_shape(chunk_shape)
+        self._set_chunk_shape(chunk_shape, use_grouped_mode)
         self.set_subregion(None)
         self._set_margin(margin)
         self._set_extra_options(extra_options)
@@ -112,7 +115,7 @@ class ChunkedPipeline:
         self._steps_name2component = {}
         self._steps_component2name = {}
 
-    def _set_chunk_shape(self, chunk_shape):
+    def _set_chunk_shape(self, chunk_shape, use_grouped_mode):
         if len(chunk_shape) != 3:
             raise ValueError("Expected chunk_shape to be a tuple of length 3 in the form (n_z, n_y, n_x)")
         self.chunk_shape = tuple(int(c) for c in chunk_shape)  # cast to int, as numpy.int64 can make pycuda crash
@@ -126,7 +129,7 @@ class ChunkedPipeline:
         self.n_angles = self.radios_shape[0]
         self.n_slices = self.radios_shape[1]
         self._grouped_processing = False
-        if self.chunk_shape[0] < len(self.process_config.rotation_angles(subsampling=False)):
+        if use_grouped_mode or self.chunk_shape[0] < len(self.process_config.rotation_angles(subsampling=False)):
             # TODO allow a certain tolerance in this case ?
             # Reconstruction is still possible (albeit less accurate) if delta is small
             self._grouped_processing = True
@@ -302,12 +305,21 @@ class ChunkedPipeline:
         process_file = options.get("process_file", None)
         if process_file is None:
             # Standard case - start pipeline from raw data
-            # ChunkReader always take a non-subsampled dictionary "files"
+            if self.process_config.nabu_config["preproc"]["detector_distortion_correction"] is None:
+                self.detector_corrector = None
+            else:
+                self.detector_corrector = DetectorDistortionProvider(
+                    detector_full_shape_vh=self.process_config.dataset_info.radio_dims[::-1],
+                    correction_type=self.process_config.nabu_config["preproc"]["detector_distortion_correction"],
+                    options=self.process_config.nabu_config["preproc"]["detector_distortion_correction_options"],
+                )
+            # ChunkReader always take a non-subsampled dictionary "files".
             self.chunk_reader = ChunkReader(
                 self._read_options["files"],
                 sub_region=self.sub_region_xz,
                 data_buffer=self.radios,
                 pre_allocate=False,
+                detector_corrector=self.detector_corrector,
                 convert_float=True,
                 binning=options["binning"],
                 dataset_subsampling=options["dataset_subsampling"],
@@ -356,10 +368,15 @@ class ChunkedPipeline:
         # Use chunk_reader.files instead of process_config.projs_indices(subsampling=True), because
         # chunk_reader might read only a subset of the files (in "grouped mode")
         self._ff_options["projs_indices"] = list(self.chunk_reader.files_subsampled.keys())
+        if self._ff_options.get("normalize_srcurrent", False):
+            a_start_idx, a_end_idx = self.sub_region[0]
+            subs = self.process_config.subsampling_factor
+            self._ff_options["radios_srcurrent"] = self._ff_options["radios_srcurrent"][a_start_idx:a_end_idx:subs]
 
         distortion_correction = None
         if self._ff_options["do_flat_distortion"]:
             self.logger.info("Flats distortion correction will be applied")
+            self.FlatFieldClass = FlatFieldDataUrls  # no GPU implementation available, force this backend
             estimation_kwargs = {}
             estimation_kwargs.update(self._ff_options["flat_distortion_params"])
             estimation_kwargs["logger"] = self.logger
@@ -376,6 +393,7 @@ class ChunkedPipeline:
             interpolation="linear",
             distortion_correction=distortion_correction,
             sub_region=self.sub_region_xz,
+            detector_corrector=self.detector_corrector,
             binning=self._ff_options["binning"],
             radios_srcurrent=self._ff_options["radios_srcurrent"],
             flats_srcurrent=self._ff_options["flats_srcurrent"],
@@ -401,6 +419,7 @@ class ChunkedPipeline:
             self.radios_shape,
             result_url=result_url,
             sub_region=self.sub_region_xz,
+            detector_corrector=self.detector_corrector,
             input_is_mlog=False,
             output_is_mlog=False,
             average_is_on_log=avg_is_on_log,
@@ -518,7 +537,7 @@ class ChunkedPipeline:
             self.reconstruction = self.FBPClass(
                 self.sinos_shape[1:],
                 angles=options["angles"],
-                rot_center=options["rotation_axis_position"],
+                rot_center=options["fbp_rotation_axis_position"],
                 filter_name=options["fbp_filter_type"],
                 slice_roi=self.process_config.rec_roi,
                 padding_mode=options["padding_type"],
@@ -573,13 +592,13 @@ class ChunkedPipeline:
             # "processing_options": self.processing_options,
             #
             "nabu_config": self.process_config.nabu_config,
-            "entry": getattr(self.dataset_info.dataset_scanner, "entry", None),
+            "entry": getattr(self.dataset_info.dataset_scanner, "entry", "entry"),
         }
         writer_extra_options = {
             "jpeg2000_compression_ratio": options["jpeg2000_compression_ratio"],
             "float_clip_values": options["float_clip_values"],
             "tiff_single_file": options.get("tiff_single_file", False),
-            "single_tiff_initialized": getattr(self.process_config, "single_tiff_initialized", False),
+            "single_output_file_initialized": getattr(self.process_config, "single_output_file_initialized", False),
         }
         self.writer = WriterManager(
             options["location"],
@@ -592,10 +611,6 @@ class ChunkedPipeline:
             histogram=("histogram" in self.processing_steps),
             extra_options=writer_extra_options,
         )
-        if options.get("tiff_single_file", False) and not getattr(
-            self.process_config, "single_tiff_initialized", False
-        ):
-            self.process_config.single_tiff_initialized = True
 
     #
     # Pipeline execution
@@ -728,7 +743,7 @@ class ChunkedPipeline:
         self.writer.write_data(data)
         self.logger.info("Wrote %s" % self.writer.fname)
         self._write_histogram()
-        self.process_config.single_tiff_initialized = True
+        self.process_config.single_output_file_initialized = True
 
     def _write_histogram(self):
         if "histogram" not in self.processing_steps: