nabu 2024.1.9__py3-none-any.whl → 2024.2.0__py3-none-any.whl

nabu/io/writer.py

@@ -5,18 +5,17 @@ from posixpath import join as posix_join
 from datetime import datetime
 import numpy as np
 from h5py import VirtualSource, VirtualLayout
-from silx.io.dictdump import dicttoh5, dicttonx
+from silx.io.dictdump import dicttoh5
 from silx.io.url import DataUrl
 
 try:
     from tomoscan.io import HDF5File
 except:
     from h5py import File as HDF5File
-from tomoscan.esrf import EDFVolume, HDF5Volume, TIFFVolume, MultiTIFFVolume, JP2KVolume, RawVolume
+from tomoscan.esrf import RawVolume
 from tomoscan.esrf.volume.jp2kvolume import has_glymur as __have_jp2k__
 from .. import version as nabu_version
-from ..utils import merged_shape, deprecation_warning
-from ..misc.utils import rescale_data
+from ..utils import merged_shape
 from .utils import convert_dict_values
 
 
@@ -39,169 +38,6 @@ class Writer:
         return self.fname
 
 
-class TomoscanNXProcessWriter(Writer):
-    """
-    A class to write Nexus file with a processing result - using tomoscan.volumes as a backend
-    """
-
-    def __init__(self, fname, entry=None, filemode="a", overwrite=False):
-        """
-        Initialize a NXProcessWriter.
-
-        Parameters
-        -----------
-        fname: str
-            Path to the HDF5 file.
-        entry: str, optional
-            Entry in the HDF5 file. Default is "entry"
-        """
-        super().__init__(fname)
-        self._set_entry(entry)
-        self._filemode = filemode
-        # TODO: notify file mode is deprecated
-        self.overwrite = overwrite
-
-    def _set_entry(self, entry):
-        self.entry = entry or "entry"
-        data_path = posix_join("/", self.entry)
-        self.data_path = data_path
-
-    def _write_npadday(self, result, volume, nx_info):
-        if result.ndim == 2:
-            result = result.reshape(1, result.shape[0], result.shape[1])
-        volume.data = result
-
-        self._update_volume_metadata(volume)
-
-        volume.save()
-        results_path = posix_join(nx_info["nx_process_path"], "results", nx_info["data_name"])
-        process_name = nx_info["process_name"]
-        process_info = nx_info["process_info"]
-        process_info.update(
-            {
-                f"{process_name}/results@NX_class": "NXdata",
-                f"{process_name}/results@signal": nx_info["data_name"],
-            }
-        )
-        if nx_info.get("is_frames_stack", True):
-            process_info.update({f"{process_name}/results@interpretation": "image"})
-        if nx_info.get("direct_access", False):
-            # prepare the direct access plots
-            process_info.update(
-                {
-                    f"{process_name}@default": "results",
-                    "@default": f"{process_name}/results",
-                }
-            )
-        return results_path
-
-    def _write_dict(self, result, volume, nx_info):
-        self._update_volume_metadata(volume)
-        volume.save_metadata()  # if result is a dictionary then we only have some metadata to be saved
-        results_path = posix_join(nx_info["nx_process_path"], "results")
-        proc_result_key = posix_join(nx_info["process_name"], "results")
-        proc_result = convert_dict_values(result, {None: "None"})
-        process_info = nx_info["process_info"]
-        process_info.update({proc_result_key: proc_result})
-        return results_path
-
-    def _write_virtual_layout(self, result, volume, nx_info):
-        # TODO: add test on tomoscan to ensure this use case is handled
-        volume.data = result
-        self._update_volume_metadata(volume)
-        volume.save()
-        results_path = posix_join(nx_info["nx_process_path"], "results", nx_info["data_name"])
-        return results_path
-
-    @staticmethod
-    def _update_volume_metadata(volume):
-        if volume.metadata is not None:
-            volume.metadata = convert_dict_values(
-                volume.metadata,
-                {None: "None"},
-            )
-
-    def write(
-        self,
-        result,
-        process_name,
-        processing_index=0,
-        config=None,
-        data_name="data",
-        is_frames_stack=True,
-        direct_access=True,
-    ) -> str:
-        """
-        Write the result in the current NXProcess group.
-
-        Parameters
-        ----------
-        result: numpy.ndarray
-            Array containing the processing result
-        process_name: str
-            Name of the processing
-        processing_index: int
-            Index of the processing (in a pipeline)
-        config: dict, optional
-            Dictionary containing the configuration.
-        """
-        entry_path = self.data_path
-        nx_process_path = "/".join([entry_path, process_name])
-
-        if config is not None:
-            config.update({"@NX_class": "NXcollection"})
-
-        nabu_process_info = {
-            "@NX_class": "NXentry",
-            f"{process_name}@NX_class": "NXprocess",
-            f"{process_name}/program": "nabu",
-            f"{process_name}/version": nabu_version,
-            f"{process_name}/date": get_datetime(),
-            f"{process_name}/sequence_index": np.int32(processing_index),
-        }
-
-        # Create HDF5Volume object with initial information
-        volume = HDF5Volume(
-            data_url=DataUrl(
-                file_path=self.fname,
-                data_path=f"{nx_process_path}/results/{data_name}",
-                scheme="silx",
-            ),
-            metadata_url=DataUrl(
-                file_path=self.fname,
-                data_path=f"{nx_process_path}/configuration",
-            ),
-            metadata=config,
-            overwrite=self.overwrite,
-        )
-        if isinstance(result, dict):
-            write_method = self._write_dict
-        elif isinstance(result, np.ndarray):
-            write_method = self._write_npadday
-        elif isinstance(result, VirtualLayout):
-            write_method = self._write_virtual_layout
-        else:
-            raise TypeError(f"'result' must be a dict, numpy array or h5py.VirtualLayout, not {type(result)}")
-        nx_info = {
-            "process_name": process_name,
-            "nx_process_path": nx_process_path,
-            "process_info": nabu_process_info,
-            "data_name": data_name,
-            "is_frames_stack": is_frames_stack,
-            "direct_access": direct_access,
-        }
-        results_path = write_method(result, volume, nx_info)
-
-        dicttonx(
-            nabu_process_info,
-            h5file=self.fname,
-            h5path=entry_path,
-            update_mode="replace",
-            mode="a",
-        )
-        return results_path
-
-
 ###################################################################################################
 ## Nabu original code for NXProcessWriter - also works for non-3D data, does not depend on tomoscan
 ###################################################################################################
@@ -534,258 +370,11 @@ def merge_hdf5_files(
         data_name=data_name,
         is_frames_stack=True,
     )
+    # pylint: disable=E0606
     if base_dir is not None and prev_cwd != getcwd():
         chdir(prev_cwd)
 
 
-class TIFFWriter(Writer):
-    def __init__(self, fname, multiframe=False, start_index=0, filemode=None, append=False, big_tiff=None):
-        """
-        Tiff writer.
-
-        Parameters
-        -----------
-        fname: str
-            Path to the output file name
-        multiframe: bool, optional
-            Whether to write all data in one single file. Default is False.
-        start_index: int, optional
-            When writing a stack of images, each image is written in a dedicated file
-            (unless multiframe is set to True).
-            In this case, the output is a series of files `filename_0000.tif`,
-            `filename_0001.tif`, etc. This parameter is the starting index for
-            file names.
-            This option is ignored when multiframe is True.
-        filemode: str, optional
-            DEPRECATED. Will be ignored. Please refer to 'append'
-        append: bool, optional
-            Whether to append data to the file rather than overwriting. Default is False.
-        big_tiff: bool, optional
-            Whether to write in "big tiff" format: https://www.awaresystems.be/imaging/tiff/bigtiff.html
-            Default is True when multiframe is True.
-            Note that default "standard" tiff cannot exceed 4 GB.
-
-        Notes
-        ------
-        If multiframe is False (default), then each image will be written in a
-        dedicated tiff file.
-        """
-        super().__init__(fname)
-        self.multiframe = multiframe
-        self.start_index = start_index
-        self.append = append
-        if big_tiff is None:
-            big_tiff = multiframe
-        if multiframe and not big_tiff:
-            # raise error ?
-            print("big_tiff was set to False while multiframe was set to True. This will probably be problematic.")
-        self.big_tiff = big_tiff
-        # Compat.
-        self.filemode = filemode
-        if filemode is not None:
-            deprecation_warning("Ignored parameter 'filemode'. Please use the 'append' parameter")
-
-    def write(self, data, *args, config=None, **kwargs):
-        ext = None
-        if not isinstance(data, np.ndarray):
-            raise TypeError(f"data is expected to be a numpy array and not {type(data)}")
-        # Single image, or multiple image in the same file
-        if self.multiframe:
-            volume = MultiTIFFVolume(
-                self.fname,
-                data=data,
-                metadata={
-                    "config": config,
-                },
-                append=self.append,
-            )
-            file_path = self.fname
-        # Multiple image, one file per image
-        else:
-            if data.ndim == 2:
-                data = data.reshape(1, data.shape[0], data.shape[1])
-            file_path, ext = path.splitext(self.fname)
-            volume = TIFFVolume(
-                path.dirname(file_path),
-                volume_basename=path.basename(file_path),
-                data=data,
-                metadata={
-                    "config": config,
-                },
-                start_index=self.start_index,
-                data_extension=ext.lstrip("."),
-                overwrite=True,
-            )
-        volume.save()
-
-
-class EDFWriter(Writer):
-    def __init__(self, fname, start_index=0, filemode="w"):
-        """
-        EDF (ESRF Data Format) writer.
-
-        Parameters
-        -----------
-        fname: str
-            Path to the output file name
-        start_index: int, optional
-            When writing a stack of images, each image is written in a dedicated file
-            In this case, the output is a series of files `filename_0000.tif`,
-            `filename_0001.edf`, etc. This parameter is the starting index for
-            file names.
-        """
-        super().__init__(fname)
-        self.filemode = filemode
-        self.start_index = start_index
-
-    def write(self, data, *args, config=None, **kwargs):
-        if not isinstance(data, np.ndarray):
-            raise TypeError(f"data is expected to be a numpy array and not {type(data)}")
-        header = {
-            "software": "nabu",
-            "data": get_datetime(),
-        }
-        if data.ndim == 2:
-            data = data.reshape(1, data.shape[0], data.shape[1])
-
-        volume = EDFVolume(path.dirname(self.fname), data=data, start_index=self.start_index, header=header)
-        volume.save()
-
-
-class JP2Writer(Writer):
-    def __init__(
-        self,
-        fname,
-        start_index=0,
-        filemode="wb",
-        psnr=None,
-        cratios=None,
-        auto_convert=True,
-        float_clip_values=None,
-        n_threads=None,
-        overwrite=False,
-        single_file=True,
-    ):
-        """
-        JPEG2000 writer. This class requires the python package `glymur` and the
-        library `libopenjp2`.
-
-        Parameters
-        -----------
-        fname: str
-            Path to the output file name
-        start_index: int, optional
-            When writing a stack of images, each image is written in a dedicated file
-            The output is a series of files `filename_0000.tif`, `filename_0001.tif`, etc.
-            This parameter is the starting index for file names.
-        psnr: list of int, optional
-            The PSNR (Peak Signal-to-Noise ratio) for each jpeg2000 layer.
-            This defines a quality metric for lossy compression.
-            The number "0" stands for lossless compression.
-        cratios: list of int, optional
-            Compression ratio for each jpeg2000 layer
-        auto_convert: bool, optional
-            Whether to automatically cast floating point data to uint16.
-            Default is True.
-        float_clip_values: tuple of floats, optional
-            If set to a tuple of two values (min, max), then each image values will be clipped
-            to these minimum and maximum values.
-        n_threads: int, optional
-            Number of threads to use for encoding. Default is the number of available threads.
-            Needs libopenjpeg >= 2.4.0.
-        """
-        super().__init__(fname)
-        if not (__have_jp2k__):
-            raise ValueError("Need glymur python package and libopenjp2 library")
-        self.n_threads = n_threads
-        # self.setup_multithread_encoding(n_threads=n_threads, what_if_not_available="ignore")
-        self.filemode = filemode
-        self.start_index = start_index
-        self.single_file = single_file
-        self.auto_convert = auto_convert
-        if psnr is not None and np.isscalar(psnr):
-            psnr = [psnr]
-        self.psnr = psnr
-        self.cratios = cratios
-        self._vmin = None
-        self._vmax = None
-        self.overwrite = overwrite
-        self.clip_float = False
-        if float_clip_values is not None:
-            self._float_clip_min, self._float_clip_max = float_clip_values
-            self.clip_float = True
-
-    def write(self, data, *args, **kwargs):
-        if not isinstance(data, np.ndarray):
-            raise TypeError(f"data is expected to be a numpy array and not {type(data)}")
-
-        if data.ndim == 2:
-            data = data.reshape(1, data.shape[0], data.shape[1])
-
-        if self.single_file and data.ndim == 3 and data.shape[0] == 1:
-            # case we will have a single file as output
-            data_url = DataUrl(
-                file_path=path.dirname(self.fname),
-                data_path=self.fname,
-                scheme=JP2KVolume.DEFAULT_DATA_SCHEME,
-            )
-            metadata_url = DataUrl(
-                file_path=path.dirname(self.fname),
-                data_path=f"{path.dirname(self.fname)}/{path.basename(self.fname)}_info.txt",
-                scheme=JP2KVolume.DEFAULT_METADATA_SCHEME,
-            )
-            volume_basename = None
-            folder = None
-            extension = None
-        else:
-            # case we need to save it as set of file
-            file_path, ext = path.splitext(self.fname)
-            data_url = None
-            metadata_url = None
-            volume_basename = path.basename(file_path)
-            folder = path.dirname(self.fname)
-            extension = ext.lstrip(".")
-
-        volume = JP2KVolume(
-            folder=folder,
-            start_index=self.start_index,
-            cratios=self.cratios,
-            psnr=self.psnr,
-            n_threads=self.n_threads,
-            volume_basename=volume_basename,
-            data_url=data_url,
-            metadata_url=metadata_url,
-            data_extension=extension,
-            overwrite=self.overwrite,
-        )
-
-        if data.dtype != np.uint16 and self.auto_convert:
-            if self.clip_float:
-                data = np.clip(data, self._float_clip_min, self._float_clip_max)
-            data = rescale_data(data, 0, 65535, data_min=self._vmin, data_max=self._vmax)
-            data = data.astype(np.uint16)
-
-        volume.data = data
-        config = kwargs.get("config", None)
-        if config is not None:
-            volume.metadata = {"config": config}
-        volume.save()
-
-
-class NPYWriter(Writer):
-    def write(self, result, *args, **kwargs):
-        np.save(self.fname, result)
-
-
-class NPZWriter(Writer):
-    def write(self, result, *args, **kwargs):
-        save_args = {"result": result}
-        config = kwargs.get("config", None)
-        if config is not None:
-            save_args["configuration"] = config
-        np.savez(self.fname, **save_args)
-
-
 class HSTVolWriter(Writer):
     """
     A writer to mimic PyHST2 ".vol" files
@@ -872,21 +461,3 @@ class HSTVolVolume(HSTVolWriter):
 
     def browse_data_files(self):
         return [self.fname]
-
-
-# Unused - kept for compat.
-Writers = {
-    "h5": NXProcessWriter,
-    "hdf5": NXProcessWriter,
-    "nx": NXProcessWriter,
-    "nexus": NXProcessWriter,
-    "npy": NPYWriter,
-    "npz": NPZWriter,
-    "tif": TIFFWriter,
-    "tiff": TIFFWriter,
-    "j2k": JP2Writer,
-    "jp2": JP2Writer,
-    "jp2k": JP2Writer,
-    "edf": EDFWriter,
-    "vol": HSTVolWriter,
-}

nabu/opencl/kernel.py

@@ -1,6 +1,9 @@
 import pyopencl.array as parray
 from pyopencl import Program, CommandQueue, kernel_work_group_info
-from ..utils import deprecation_warning
+from ..utils import (
+    deprecation_warning,
+    catch_warnings,
+)  # TODO use warnings.catch_warnings once python < 3.11 is dropped
 from ..processing.kernel_base import KernelBase
 
 
@@ -34,9 +37,16 @@ class OpenCLKernel(KernelBase):
         filename=None,
         src=None,
         automation_params=None,
+        silent_compilation_warnings=False,
         **build_kwargs,
     ):
-        super().__init__(kernel_name, filename=filename, src=src, automation_params=automation_params)
+        super().__init__(
+            kernel_name,
+            filename=filename,
+            src=src,
+            automation_params=automation_params,
+            silent_compilation_warnings=silent_compilation_warnings,
+        )
         if queue is not None:
             self.ctx = queue.context
             self.queue = queue
@@ -49,7 +59,8 @@ class OpenCLKernel(KernelBase):
     def compile_kernel_source(self, kernel_name, build_kwargs):
         self.build_kwargs = build_kwargs
         self.kernel_name = kernel_name
-        self.program = Program(self.ctx, self.src).build(**self.build_kwargs)
+        with catch_warnings(action=("ignore" if self.silent_compilation_warnings else None)):  # pylint: disable=E1123
+            self.program = Program(self.ctx, self.src).build(**self.build_kwargs)
 
     def get_kernel(self):
         self.kernel = None

nabu/opencl/processing.py

@@ -14,6 +14,7 @@ else:
     dtype_to_ctype = MissingComponentError("pyopencl")
 
 
+# pylint: disable=E0606
 class OpenCLProcessing(ProcessingBase):
     array_class = OpenCLArray
     dtype_to_ctype = dtype_to_ctype
@@ -44,6 +45,13 @@ class OpenCLProcessing(ProcessingBase):
                 queue_init_kwargs = {"properties": cl.command_queue_properties.PROFILING_ENABLE}
             queue = cl.CommandQueue(self.ctx, **queue_init_kwargs)
         self.queue = queue
+        dev_types = {
+            cl.device_type.CPU: "cpu",
+            cl.device_type.GPU: "gpu",
+            cl.device_type.ACCELERATOR: "accelerator",
+            -1: "unknown",
+        }
+        self.device_type = dev_types.get(self.ctx.devices[0].type, "unknown")
 
     # TODO push_context, pop_context ?
 

nabu/pipeline/config_validators.py

@@ -445,9 +445,12 @@ def fbp_filter_name_validator(val):
 
 
 @validator
-def iterative_method_name_validator(val):
+def reconstruction_implementation_validator(val):
     return name_range_checker(
-        val, set(iterative_methods.values()), "iterative methods name", replacements=iterative_methods
+        val,
+        set(reco_implementations.values()),
+        "Reconstruction method implementation",
+        replacements=reco_implementations,
     )
 
 

nabu/pipeline/datadump.py

@@ -50,7 +50,7 @@ class DataDumpManager:
         self._z_sub_region = self.sub_region[1]
         self.z_min = self._z_sub_region[0]
         self.margin = get_subregion(margin, ndim=2)  # ((U, D), (L, R))
-        self.margin_up = self.margin[0][0]
+        self.margin_up = self.margin[0][0] or 0
         self.start_index = self.z_min + self.margin_up
         self.delta_z = self._z_sub_region[-1] - self._z_sub_region[-2]
 
@@ -131,8 +131,8 @@ class DataDumpManager:
         relative_end_z = relative_start_z + self.delta_z
         # When using binning, every step after "read" results in smaller-sized data.
         # Therefore dumped data has shape (ceil(n_angles/subsampling), n_z//binning_z, n_x//binning_x)
-        relative_start_z //= self.process_config.binning_x
-        relative_end_z //= self.process_config.binning_x
+        relative_start_z //= self.process_config.binning_z
+        relative_end_z //= self.process_config.binning_z
         # (n_angles, n_z, n_x)
         subregion = (None, None, relative_start_z, relative_end_z, None, None)
         return subregion
@@ -146,7 +146,7 @@ class DataDumpManager:
         )
         # TODO check
 
-    def dump_data_to_file(self, step_name, data):
+    def dump_data_to_file(self, step_name, data, crop_margin=False):
         if step_name not in self.data_dump:
             return
         writer = self.data_dump[step_name]
@@ -154,7 +154,14 @@ class DataDumpManager:
         if __has_pycuda__:
             if isinstance(data, garray.GPUArray):
                 data = data.get()
-        writer.write_data(data)
+
+        margin_up = self.margin[0][0] or None
+        margin_down = self.margin[0][1] or None
+        margin_down = -margin_down if margin_down is not None else None  # pylint: disable=E1130
+        if crop_margin and (margin_up is not None or margin_down is not None):
+            data = data[:, margin_up:margin_down, :]
+        metadata = {"dump_sub_region": {"sub_region": self.sub_region, "margin": self.margin}}
+        writer.write_data(data, metadata=metadata)
 
     def __repr__(self):
         res = "%s(%s, margin=%s)" % (self.__class__.__name__, str(self.sub_region), str(self.margin))