nabu 2023.2.1__py3-none-any.whl → 2024.1.0rc3__py3-none-any.whl

nabu/pipeline/helical/helical_chunked_regridded_cuda.py

@@ -1,16 +1,8 @@
-from math import ceil
 import numpy as np
-from ...utils import deprecated
-from ...preproc.flatfield_cuda import CudaFlatFieldDataUrls
-from ...preproc.shift import VerticalShift
 from ...preproc.shift_cuda import CudaVerticalShift
 
-from ...preproc.phase_cuda import CudaPaganinPhaseRetrieval
 from ...reconstruction.sinogram_cuda import CudaSinoBuilder, CudaSinoNormalization
-from ...reconstruction.sinogram import SinoBuilder, SinoNormalization
-from ...misc.unsharp_cuda import CudaUnsharpMask
-from ...misc.rotation_cuda import CudaRotation
-from ...misc.histogram_cuda import CudaPartialHistogram
+from ...processing.histogram_cuda import CudaPartialHistogram
 from .fbp import BackprojectorHelical
 
 
@@ -21,8 +13,7 @@ try:
 except:
     HierarchicalBackprojector = None
 
-from ...cuda.utils import get_cuda_context, __has_pycuda__, __pycuda_error_msg__, replace_array_memory
-from ..utils import pipeline_step
+from ...cuda.utils import get_cuda_context, __has_pycuda__, __pycuda_error_msg__
 from .helical_chunked_regridded import HelicalChunkedRegriddedPipeline
 
 if __has_pycuda__:
@@ -53,6 +44,7 @@ class CudaHelicalChunkedRegriddedPipeline(HelicalChunkedRegriddedPipeline):
         reading_granularity=10,
         span_info=None,
         num_weight_radios_per_app=1000,
+        diag_zpro_run=0,
     ):
         self._init_cuda(cuda_options)
         super().__init__(
@@ -63,6 +55,7 @@ class CudaHelicalChunkedRegriddedPipeline(HelicalChunkedRegriddedPipeline):
             phase_margin=phase_margin,
             reading_granularity=reading_granularity,
             span_info=span_info,
+            diag_zpro_run=diag_zpro_run,
         )
         self._register_callbacks()
 

nabu/pipeline/helical/helical_reconstruction.py

@@ -1,23 +1,24 @@
-from os.path import join, isfile, basename, dirname
+from os.path import join, isfile, dirname
 from math import ceil
-import gc
 from time import time
-from psutil import virtual_memory
-from silx.io import get_data
-from silx.io.url import DataUrl
 import numpy as np
-import math
 import copy
 from ...resources.logger import LoggerOrPrint
-from ...io.writer import merge_hdf5_files, NXProcessWriter
+from ...io.writer import merge_hdf5_files
 from ...cuda.utils import collect_cuda_gpus
-from ...preproc.phase import compute_paganin_margin
-from ...misc.histogram import PartialHistogram, add_last_bin, hist_as_2Darray
-from .span_strategy import SpanStrategy
+
+try:
+    import nabuxx
+
+    SpanStrategy = nabuxx.span_strategy.SpanStrategy
+except:
+    logger_tmp = LoggerOrPrint(None)
+    logger_tmp.info("Nabuxx not available. Loading python implementation for SpanStrategy")
+    from .span_strategy import SpanStrategy
 
 from .helical_chunked_regridded_cuda import CudaHelicalChunkedRegriddedPipeline
 
-from ..fullfield.reconstruction import collect_cuda_gpus, variable_idxlen_sort, FullFieldReconstructor
+from ..fullfield.reconstruction import collect_cuda_gpus, FullFieldReconstructor
 
 avail_gpus = collect_cuda_gpus() or {}
 
@@ -62,12 +63,20 @@ class HelicalReconstructorRegridded:
            - "max_chunk_size": None,
            - "phase_margin": None,
            - "dry_run": 0,
+           - "diag_zpro_run": 0,
         """
         self.logger = LoggerOrPrint(logger)
         self.process_config = process_config
         self._set_extra_options(extra_options)
         self._get_resources()
-        self._margin_v, self._margin_h = self._compute_phase_margin()
+
+        ### intrication problem: this is used in fullfield's compute_margin to clamp the margin but not used by the present pipeline
+        ### Set it to a big number that will never clamp
+        self.n_z = 10000000  # a big number
+        self.n_x = 10000000  # a big number
+        ###
+        self._compute_margin()
+        # self._margin_v, self._margin_h = self._compute_phase_margin()
 
         self._setup_span_info()
         self._compute_max_chunk_size()
@@ -87,6 +96,7 @@ class HelicalReconstructorRegridded:
             "max_chunk_size": None,
             "phase_margin": None,
             "dry_run": 0,
+            "diag_zpro_run": 0,
         }
         advanced_options.update(extra_options)
         self.extra_options = advanced_options
@@ -95,8 +105,15 @@ class HelicalReconstructorRegridded:
         self.use_phase_margin = self.extra_options["use_phase_margin"]
 
         self.dry_run = self.extra_options["dry_run"]
+        self.diag_zpro_run = self.extra_options["diag_zpro_run"]
 
         self._do_histograms = self.process_config.nabu_config["postproc"]["output_histogram"]
+
+        if self.diag_zpro_run:
+            self.process_config.processing_options.get("phase", None)
+            self._do_histograms = False
+            self.reading_granularity = 10
+
         self._histogram_merged = False
         self._span_info = None
 
@@ -145,13 +162,23 @@ class HelicalReconstructorRegridded:
                         break
         elif z_fract_min != 0.0 or z_fract_max != 0.0:
             z_start, z_max = (self._span_info.get_doable_span()).view_heights_minmax
-            self.z_min = int(round(z_start * (1 - z_fract_min) + z_max * z_fract_min))
-            self.z_max = int(round(z_start * (1 - z_fract_max) + z_max * z_fract_max)) + 1
+
+            # the meaming of z_min and z_max is: position in slices units from the
+            # first available slice and in the direction of the scan
+            self.z_min = int(round(z_start * (0 - z_fract_min) + z_max * z_fract_min))
+            self.z_max = int(round(z_start * (0 - z_fract_max) + z_max * z_fract_max)) + 1
+
+    def _compute_translations_margin(self):
+        return 0, 0
+
+    def _compute_cone_overlap(self):
+        return 0, 0
 
     _get_resources = FullFieldReconstructor._get_resources
     _get_memory = FullFieldReconstructor._get_memory
     _get_gpu = FullFieldReconstructor._get_gpu
     _compute_phase_margin = FullFieldReconstructor._compute_phase_margin
+    _compute_margin = FullFieldReconstructor._compute_margin
     _compute_unsharp_margin = FullFieldReconstructor._compute_unsharp_margin
     _print_tasks = FullFieldReconstructor._print_tasks
     _instantiate_pipeline_if_necessary = FullFieldReconstructor._instantiate_pipeline_if_necessary
@@ -180,6 +207,14 @@ class HelicalReconstructorRegridded:
 
         user_max_chunk_size = self.extra_options["max_chunk_size"]
 
+        if self.diag_zpro_run:
+            if user_max_chunk_size is not None:
+                user_max_chunk_size = min(
+                    user_max_chunk_size, max(self.process_config.dataset_info.radio_dims[1] // 4, 10)
+                )
+            else:
+                user_max_chunk_size = max(self.process_config.dataset_info.radio_dims[1] // 4, 10)
+
         self.cpu_max_chunk_size = self.estimate_chunk_size(
             cpu_mem, self.process_config, chunk_step=1, user_max_chunk_size=user_max_chunk_size
         )
@@ -219,12 +254,14 @@ class HelicalReconstructorRegridded:
                 reading_granularity=self.reading_granularity,
                 margin_v=self._margin_v,
                 span_info=self._span_info,
+                diag_zpro_run=self.diag_zpro_run,
             )
 
             required_mem_GB = required_mem / 1e9
             if required_mem_GB > available_memory_GB:
                 break
             last_good_chunk_size = chunk_size
+
             if user_max_chunk_size is not None and chunk_size > user_max_chunk_size:
                 last_good_chunk_size = user_max_chunk_size
                 break
@@ -270,7 +307,7 @@ class HelicalReconstructorRegridded:
                 """
                 raise ValueError(message)
 
-            if self._span_info.z_pix_per_proj[-1] > self._span_info.z_pix_per_proj[0]:
+            if self._span_info.z_pix_per_proj[-1] >= self._span_info.z_pix_per_proj[0]:
                 my_z_min = z_start + self.z_min
                 my_z_end = z_start + self.z_max
             else:
@@ -283,7 +320,8 @@ class HelicalReconstructorRegridded:
         print("my_z_min my_z_end ", my_z_min, my_z_end)
         if my_z_min >= my_z_end:
             message = f""" The requested vertical span, after translation to absolute doable heights would be {my_z_min, my_z_end}
-            is not  doable (start>=end)
+            is not  doable (start>=end).
+            Scans are often shorter than expected ThereFore :  CONSIDER TO INCREASE angular_tolerance_steps
             """
             raise ValueError(message)
 
@@ -406,7 +444,6 @@ class HelicalReconstructorRegridded:
             phase_margin_pix=self._margin_v,
             projection_angles_deg=angles_deg,
             pixel_size_mm=self.process_config.dataset_info.pixel_size * 1.0e-3,  # micron to mm
-            logger=self.logger,
             require_redundancy=(redundancy_angle_deg > 0),
             angular_tolerance_steps=self.process_config.nabu_config["reconstruction"]["angular_tolerance_steps"],
         )
@@ -430,7 +467,8 @@ class HelicalReconstructorRegridded:
             logger=self.logger,
             phase_margin=task["phase_margin"],
             reading_granularity=self.reading_granularity,
-            span_info=self._span_info
+            span_info=self._span_info,
+            diag_zpro_run=self.diag_zpro_run
             # cuda_options=self.cuda_options
         )
 

nabu/pipeline/helical/span_strategy.py

@@ -186,7 +186,7 @@ class SpanStrategy:
 
     def get_informative_string(self):
         doable_span_v = self.get_doable_span()
-        if self.z_pix_per_proj[-1] > self.z_pix_per_proj[-1]:
+        if self.z_pix_per_proj[-1] >= self.z_pix_per_proj[-1]:
             direction = "ascending"
         else:
             direction = "descending"

nabu/pipeline/helical/tests/test_accumulator.py

@@ -95,11 +95,14 @@ class TestGriddedAccumulator:
             diagnostic_radios=reconstruction_space.diagnostic_radios,
             diagnostic_weights=reconstruction_space.diagnostic_weights,
             diagnostic_angles=reconstruction_space.diagnostic_proj_angle,
+            diagnostic_searched_angles_rad_clipped=reconstruction_space.diagnostic_searched_angles_rad_clipped,
+            diagnostic_zpix_transl=reconstruction_space.diagnostic_zpix_transl,
             dark=self.dark,
             flat_indexes=[0, 7501],
             flats=self.flats,
             weights=self.weights_field,
             double_flat=self.double_flat,
+            diag_zpro_run=0,
         )
 
         # splitting in sub ranges of 100 projections
@@ -150,4 +153,5 @@ class TestGriddedAccumulator:
             np.array((subr_start_z, subr_end_z), "i"),
             np.array((dtasrc_start_z, dtasrc_end_z), "i"),
             data_raw,
+            sub_total_prange_slice,
         )

nabu/pipeline/params.py

@@ -107,15 +107,25 @@ cor_methods = {
     "auto": "centered",
     "centered": "centered",
     "global": "global",
-    "sliding-window": "sliding-window",
+    "sino sliding window": "sino-sliding-window",
+    "sino-sliding-window": "sino-sliding-window",
     "sliding window": "sliding-window",
-    "growing-window": "growing-window",
+    "sliding-window": "sliding-window",
+    "sino growing window": "sino-growing-window",
+    "sino-growing-window": "sino-growing-window",
     "growing window": "growing-window",
+    "growing-window": "growing-window",
     "sino-coarse-to-fine": "sino-coarse-to-fine",
     "composite-coarse-to-fine": "composite-coarse-to-fine",
     "near": "composite-coarse-to-fine",
+    "fourier-angles": "fourier-angles",
+    "fourier angles": "fourier-angles",
+    "fourier-angle": "fourier-angles",
+    "fourier angle": "fourier-angles",
+    "octave-accurate": "octave-accurate",
 }
 
+
 tilt_methods = {
     "1d-correlation": "1d-correlation",
     "1dcorrelation": "1d-correlation",
@@ -128,6 +138,11 @@ rings_methods = {
     "none": None,
     "": None,
     "munch": "munch",
+    "mean-subtraction": "mean-subtraction",
+    "mean_subtraction": "mean-subtraction",
+    "mean-division": "mean-division",
+    "mean_division": "mean-division",
+    "vo": "vo",
 }
 
 detector_distortion_correction_methods = {"none": None, "": None, "identity": "identity", "map_xz": "map_xz"}
@@ -140,3 +155,9 @@ radios_rotation_mode = {
     "chunks": "chunk",
     "full": "full",
 }
+
+exclude_projections_type = {
+    "indices": "indices",
+    "angular_range": "angular_range",
+    "angles": "angles",
+}

nabu/pipeline/processconfig.py

@@ -1,9 +1,9 @@
 import os
 from .config import parse_nabu_config_file
-from ..utils import deprecation_warning, is_writeable
+from ..utils import is_writeable
 from ..resources.logger import Logger, PrinterLogger
 from .config import validate_config
-from ..resources.dataset_analyzer import analyze_dataset, _tomoscan_has_nxversion
+from ..resources.dataset_analyzer import analyze_dataset
 from .estimators import DetectorTiltEstimator
 
 
@@ -21,8 +21,6 @@ class ProcessConfigBase:
         conf_fname=None,
         conf_dict=None,
         dataset_info=None,
-        checks=True,
-        remove_unused_radios=True,
         create_logger=False,
     ):
         """
@@ -139,11 +137,8 @@ class ProcessConfigBase:
             extra_options = {
                 "exclude_projections": self.nabu_config["dataset"]["exclude_projections"],
                 "hdf5_entry": self.nabu_config["dataset"]["hdf5_entry"],
+                "nx_version": self.nabu_config["dataset"]["nexus_version"],
             }
-            if _tomoscan_has_nxversion:  # legacy - should become "if True" soon
-                extra_options["nx_version"] = self.nabu_config["dataset"]["nexus_version"]
-            else:
-                self.logger.warning("Cannot use 'nx_version' for browsing dataset: need tomoscan > 0.6.0")
             self.dataset_info = analyze_dataset(
                 self.nabu_config["dataset"]["location"], extra_options=extra_options, logger=self.logger
             )

nabu/pipeline/tests/test_chunk_reader.py

@@ -0,0 +1,78 @@
+import numpy as np
+import pytest
+from nabu.io.utils import get_compacted_dataslices
+from nabu.resources.dataset_analyzer import HDF5DatasetAnalyzer
+from nabu.testutils import compare_arrays, __do_long_tests__, get_file
+from nabu.io.reader import ChunkReader
+
+
+@pytest.fixture(scope="class")
+def bootstrap(request):
+    cls = request.cls
+
+    cls.dataset_fname = get_file("bamboo_reduced.nx")
+    cls.tol = 1e-7
+
+
+def get_compacted_dataslices_as_sorted_tuples(reader):
+    slice_to_tuple = lambda s: (s.start, s.stop, s.step)
+    data_slices_tuples = list(
+        set(
+            [
+                slice_to_tuple(u.data_slice())
+                for u in get_compacted_dataslices(
+                    reader.files, subsampling=reader.dataset_subsampling, begin=reader._files_begin_idx
+                ).values()
+            ]
+        )
+    )
+    return sorted(data_slices_tuples, key=lambda t: t[0])
+
+
+@pytest.mark.skipif(not (__do_long_tests__), reason="Use __do_long_tests__ for this test")
+@pytest.mark.usefixtures("bootstrap")
+class TestChunkReader:
+    def test_subsampling(self):
+        """
+        Test reading data from even/odd projections
+        """
+        dataset_info = HDF5DatasetAnalyzer(self.dataset_fname)
+
+        # Read all projections, then only even-numbered ones, then odd-numbered ones
+        # Use the same object to hopefully use less memory (through garbage collection)
+        reader = ChunkReader(dataset_info.projections, dataset_subsampling=None)
+        reader.load_data()
+        first_sino_all = reader.data[:, 0, :].copy()
+        compacted_dataslices_all = get_compacted_dataslices_as_sorted_tuples(reader)
+
+        reader = ChunkReader(dataset_info.projections, dataset_subsampling=(2, 0))
+        reader.load_data()
+        first_sino_even = reader.data[:, 0, :].copy()
+        compacted_dataslices_even = get_compacted_dataslices_as_sorted_tuples(reader)
+
+        reader = ChunkReader(dataset_info.projections, dataset_subsampling=(2, 1))
+        reader.load_data()
+        first_sino_odd = reader.data[:, 0, :].copy()
+        compacted_dataslices_odd = get_compacted_dataslices_as_sorted_tuples(reader)
+
+        # from spire.utils import ims
+        # ims([first_sino_all[::2], first_sino_even, first_sino_all[::2]*1. - first_sino_even])
+        # ims([first_sino_all[::2], first_sino_even, first_sino_all[1::2]*1. - first_sino_odd])
+
+        # Check that the compacted data slices are correct
+        assert len(compacted_dataslices_all) == len(compacted_dataslices_even) == len(compacted_dataslices_odd)
+        for data_slice_all, data_slice_even, data_slice_odd in zip(
+            compacted_dataslices_all, compacted_dataslices_even, compacted_dataslices_odd
+        ):
+            indices_all = np.arange(100000)[slice(*data_slice_all)]
+            indices_even = np.arange(100000)[slice(*data_slice_even)]
+            indices_odd = np.arange(100000)[slice(*data_slice_odd)]
+            assert indices_all[::2].size == indices_even.size
+            assert np.allclose(indices_all[::2], indices_even)
+            assert indices_all[1::2].size == indices_odd.size
+            assert np.allclose(indices_all[1::2], indices_odd)
+
+        is_close, max_err = compare_arrays(first_sino_all[::2, :], first_sino_even, self.tol, return_residual=True)
+        assert is_close, "ChunkReader: something wrong when subsampling for even projections. max_err=%.3e" % max_err
+        is_close, max_err = compare_arrays(first_sino_all[1::2, :], first_sino_odd, self.tol, return_residual=True)
+        assert is_close, "ChunkReader: something wrong when subsampling for odd projections. max_err=%.3e" % max_err

nabu/pipeline/tests/test_estimators.py

@@ -1,8 +1,18 @@
+import os
 import pytest
 import numpy as np
-from nabu.testutils import utilstest
+from nabu.testutils import utilstest, __do_long_tests__
 from nabu.resources.dataset_analyzer import HDF5DatasetAnalyzer
+from nabu.resources.dataset_analyzer import analyze_dataset
+from nabu.resources.utils import extract_parameters
 from nabu.pipeline.estimators import CompositeCOREstimator
+from nabu.pipeline.config import parse_nabu_config_file
+from nabu.pipeline.estimators import SinoCORFinder, CORFinder
+
+
+#
+# Test CoR estimation with "composite-coarse-to-fine" (aka "near" in the legacy system vocable)
+#
 
 
 @pytest.fixture(scope="class")
@@ -16,9 +26,10 @@ def bootstrap(request):
     cls.cor_pix = 1321.625
     cls.abs_tol = 0.0001
     cls.dataset_info = HDF5DatasetAnalyzer(dataset_downloaded_path)
-    cls.cor_options = """side="near"; near_pos = 300.0;  near_width = 20.0 """
+    cls.cor_options = extract_parameters("""side="near"; near_pos = 300.0;  near_width = 20.0 """, sep=";")
 
 
+@pytest.mark.skipif(not (__do_long_tests__), reason="Need NABU_LONG_TESTS=1 for this test")
 @pytest.mark.usefixtures("bootstrap")
 class TestCompositeCorFinder:
     def test(self):
@@ -29,3 +40,110 @@ class TestCompositeCorFinder:
         cor_position = cor_finder.find_cor()
         message = "Computed CoR %f " % cor_position + " and real CoR %f do not coincide" % self.cor_pix
         assert np.isclose(self.cor_pix, cor_position, atol=self.abs_tol), message
+
+
+@pytest.fixture(scope="class")
+def bootstrap_bamboo_reduced(request):
+    cls = request.cls
+    cls.abs_tol = 0.2
+    # Dataset without estimated_cor_frm_motor (non regression test)
+    dataset_relpath = os.path.join("bamboo_reduced.nx")
+    dataset_downloaded_path = utilstest.getfile(dataset_relpath)
+    conf_relpath = os.path.join("bamboo_reduced.conf")
+    conf_downloaded_path = utilstest.getfile(conf_relpath)
+    cls.ds_std = analyze_dataset(dataset_downloaded_path)
+    cls.conf_std = parse_nabu_config_file(conf_downloaded_path)
+
+    # Dataset with estimated_cor_frm_motor
+    dataset_relpath = os.path.join("bamboo_reduced_bliss.nx")
+    dataset_downloaded_path = utilstest.getfile(dataset_relpath)
+    conf_relpath = os.path.join("bamboo_reduced_bliss.conf")
+    conf_downloaded_path = utilstest.getfile(conf_relpath)
+    cls.ds_bliss = analyze_dataset(dataset_downloaded_path)
+    cls.conf_bliss = parse_nabu_config_file(conf_downloaded_path)
+
+
+@pytest.mark.skipif(not (__do_long_tests__), reason="need environment variable NABU_LONG_TESTS=1")
+@pytest.mark.usefixtures("bootstrap_bamboo_reduced")
+class TestCorNearPos:
+    true_cor = 339.486
+
+    def test_cor_sliding_standard(self):
+        cor_options = extract_parameters(self.conf_std["reconstruction"].get("cor_options", None), sep=";")
+        finder = CORFinder("sliding-window", self.ds_std, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+        cor_options.update({"side": "from_file"})
+        finder = CORFinder("sliding-window", self.ds_std, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+        cor_options.update({"side": "center"})
+        finder = CORFinder("sliding-window", self.ds_std, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+    def test_cor_fourier_angles_standard(self):
+        cor_options = extract_parameters(self.conf_std["reconstruction"].get("cor_options", None), sep=";")
+        finder = SinoCORFinder("fourier-angles", self.ds_std, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+        # Checks that it works though no data in NX
+        cor_options.update({"side": "from_file"})
+        finder = SinoCORFinder("fourier-angles", self.ds_std, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+        cor_options.update({"side": "center"})
+        finder = SinoCORFinder("fourier-angles", self.ds_std, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+    def test_cor_sliding_bliss(self):
+        cor_options = extract_parameters(self.conf_bliss["reconstruction"].get("cor_options", None), sep=";")
+        finder = CORFinder("sliding-window", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        print(message)
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+        cor_options.update({"side": "from_file"})
+        finder = CORFinder("sliding-window", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        print(message)
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+        cor_options.update({"side": "center"})
+        finder = CORFinder("sliding-window", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+    def test_cor_fourier_angles_bliss(self):
+        cor_options = extract_parameters(self.conf_bliss["reconstruction"].get("cor_options", None), sep=";")
+        finder = SinoCORFinder("fourier-angles", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+        # Checks that it works though no data in NX
+        cor_options.update({"side": "from_file"})
+        finder = SinoCORFinder("fourier-angles", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message
+
+        cor_options.update({"side": "center"})
+        finder = SinoCORFinder("fourier-angles", self.ds_bliss, do_flatfield=True, cor_options=cor_options)
+        cor = finder.find_cor()
+        message = f"Computed CoR {cor} and expected CoR {self.true_cor} do not coincide. Near_pos options was set to {cor_options.get('near_pos',None)}."
+        assert np.isclose(self.true_cor, cor, atol=self.abs_tol), message

nabu/pipeline/utils.py

@@ -1,4 +1,7 @@
 from ..utils import deprecated_class
+from .config_validators import str2bool
+from dataclasses import dataclass
+import os
 
 #
 # Decorators and callback mechanism
@@ -89,3 +92,25 @@ from .writer import WriterManager
 WriterConfigurator = deprecated_class("WriterConfigurator moved to nabu.pipeline.writer.WriterManager", do_print=True)(
     WriterManager
 )
+
+
+@dataclass
+class EnvSettings:
+    """This class centralises the definitions, possibly documentation, and access to environnt variable
+    driven settings.
+    It is meant to be used in the following way:
+          from nabu.utils import nabu_env_settings
+          if not nabu_env_settings.skip_tomoscan_checks:
+                do something
+    """
+
+    skip_tomoscan_checks: bool = False
+
+
+def _get_nabu_environment_variables():
+    nabu_env_settings = EnvSettings()
+    nabu_env_settings.skip_tomoscan_checks = str2bool(os.getenv("SKIP_TOMOSCAN_CHECK", "0"))
+    return nabu_env_settings
+
+
+nabu_env_settings = _get_nabu_environment_variables()

nabu/pipeline/writer.py

@@ -124,6 +124,7 @@ class WriterManager:
             if self.file_format == "hdf5":
                 writer_kwargs["data_path"] = self.metadata.get("entry", "entry")
                 writer_kwargs["process_name"] = self.metadata.get("process_name", "reconstruction")
+                writer_kwargs["create_subfolder"] = self.extra_options.get("create_subfolder", True)
             elif self.file_format == "jp2":
                 writer_kwargs["cratios"] = self.metadata.get("jpeg2000_compression_ratio", None)
                 writer_kwargs["clip_values"] = self.metadata.get("float_clip_values", None)
@@ -135,6 +136,7 @@ class WriterManager:
                 ),
                 "overwrite": self.overwrite,
                 "append": self.extra_options.get("single_output_file_initialized", False),
+                "hst_metadata": self.extra_options.get("raw_vol_metadata", {}),
             }
         else:
             raise ValueError("Unsupported file format: %s" % self.file_format)

nabu/preproc/ccd_cuda.py

@@ -1,9 +1,11 @@
-from typing import Union
 import numpy as np
 from ..preproc.ccd import CCDFilter, Log
-from ..cuda.kernel import CudaKernel
-from ..cuda.medfilt import MedianFilter
+from ..processing.medfilt_cuda import MedianFilter
 from ..utils import get_cuda_srcfile, updiv, deprecated_class
+from ..cuda.utils import __has_pycuda__
+
+if __has_pycuda__:
+    from ..cuda.kernel import CudaKernel
 
 # COMPAT.
 from .flatfield_cuda import (
@@ -27,11 +29,11 @@ FlatFieldDataUrls = deprecated_class(
 class CudaCCDFilter(CCDFilter):
     def __init__(
         self,
-        radios_shape: tuple,
-        correction_type: str = "median_clip",
-        median_clip_thresh: float = 0.1,
+        radios_shape,
+        correction_type="median_clip",
+        median_clip_thresh=0.1,
         abs_diff=False,
-        cuda_options: Union[dict, None] = None,
+        cuda_options=None,
     ):
         """
         Initialize a CudaCCDCorrection instance.