nabu 2024.1.0rc3__py3-none-any.whl → 2024.1.1__py3-none-any.whl

build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/nabu/app/diag_to_rot.py

@@ -0,0 +1,449 @@
+from .. import version
+from os import environ
+
+import argparse
+import shutil
+import os
+import sys
+import re
+import h5py
+import numpy as np
+
+
+from ..resources.logger import LoggerOrPrint
+from .utils import parse_params_values
+from .cli_configs import DiagToRotConfig
+
+from ..utils import get_available_threads
+
+from nxtomo.application.nxtomo import NXtomo
+
+from multiprocessing import Pool
+
+import h5py
+from nabu.utils import DictToObj
+from nabu.pipeline.estimators import oversample
+from scipy.ndimage import gaussian_filter
+from silx.io.dictdump import h5todict
+
+
+"""
+  The operations here below rely on diag objects which are found in the result of a nab-helical run with the diag_zpro_run set to a number > 0
+  They are found in the configuration section of the nabu output, in several sequential dataset, with hdf5 dataset keys 
+  which are 0,1,2.... corresponding to all the z-windows ( chunck) for which we have collected contributions at different angles
+  which are nothing else but pieces of ready to use preprocessed radio , and this for different angles.
+  In other words redundant contributions conccurring at the same prepocessed radiography but comming from different moment 
+  of the helical scan are kept separate.
+  By forming pairs theta, theta +180
+  These Pairs should coincide on an overlapping region after flipping one, 
+  where they both have signal ( part of them can be dark if the detector is out of view above or below)
+  For each key there is a sequence of radio, the corresponding sequence of weights map, the corresponding z translation, and angles
+
+  The number passed to diag_zpro_run object, is >1, and is interpreted by the diagnostic collection run 
+  as the number of wished collected angles between 0 and 180. Lets call it num_0_180
+
+  The collection is not done here. Here we exploit the result of a previous collection to deduce, looking at the correlations, the cor
+
+  An example of collection is this :
+    
+   |_____ diagnostics
+   |               |  
+                  |__ 0
+                  |    |_ radios       (4*num_0_180, chunky, chunkx) 
+                  |    |
+                  |    |_ weights      (4*num_0_180, chunky, chunkx) 
+                  |    |
+                  |    |_ angles       ( 4*num_0_180,)
+                       |
+                       |_ searched_rad ( 2*num_0_180,)    these are all the searched angles between 0 and 360 in radians
+                       |
+                       |_ zmm_trans    (  4*num_0_180,)    the z translation in mm
+                       |
+                       |_ zpix_transl  (  4*num_0_180,)    the z translation in pix
+                       |
+                       |_ pixes_size_mm  scalar
+
+
+   Here we follow the evolution of the rotation angle along the scan.
+   The final result can be left in its detailed form, giving the found cor at every analysed scan position,
+   or the result of the interpolation, giving the cor at the two extremal position of z_translation.
+
+"""
+
+
+def transform_images(diag, args):
+    """
+    Filter and transform the radios and the weights.
+
+    Filter the radios, and oversample them along the horizontal line.
+    The method in general is similar to the composite cor finding.
+    Several overlapping positions are used to match redundant contributions at
+    different rotation stages ( theta and theta+180).
+    But beforehand it is beneficial to remove low spatial frequencies.
+    And we do oversampling on the fly.
+
+    Parameters:
+       diag: object
+         used member of diag are radios and weights
+       args: object
+         its member are the application parameters. Here we use only:
+             low_pass, high_pass, ovs ( oversampling factor for the horisontal dimension )
+
+
+
+
+    """
+
+    diag.radios[:] = (diag.radios / diag.weights).astype("f")
+
+    new_radios = []
+
+    for ima in diag.radios:
+        ima = gaussian_filter(ima, [0, args.low_pass], mode="nearest")
+        ima = ima - gaussian_filter(ima, [0, args.high_pass], mode="nearest")
+        new_radios.append(ima)
+
+    diag.radios = [oversample(ima, [1, args.ovs]).astype("f") for ima in new_radios]
+
+    diag.weights = [oversample(ima, [1, args.ovs]).astype("f") for ima in diag.weights]
+
+
+def total_merit_list(arg_tuple):
+    """
+    builds three lists : all_merits, all_energies, all_z_transl
+
+    For every pair (theta, theta+180 ) add an item to the list which contains:
+    for "all_merits" a list of merit, one for every overlap in the overlap_list argument,
+    for "all_energies", same logic, but calculating the implied energy, implied in the calculation of the merit,
+    for "all_z_transl" we add the averaged z_transl for the considered pair
+    Parameters:
+       diag: object
+         used member of diag are radios, weights and zpix_transl
+       args: object
+          containing the application parameters.
+          Its used members are ovs, high_pass, low_pass
+    """
+
+    (diag, overlap_list, args) = arg_tuple
+
+    orig_sy, ovsd_sx = diag.radios[0].shape
+
+    all_merits = []
+    all_energies = []
+    all_z_transls = []
+
+    # the following two lines are in accordance with the nabu collection layout for diagos
+    # there are n_angles_pi in [0,180[, and then again the same number of possibly valid radios
+    # (check for nan in z translation) in [180,360[, [360,540[, 540,720[
+    n_angles_pi = len(diag.radios) // 4
+    n_angles_2pi = len(diag.radios) // 2
+
+    # check for (theta, theta+180 )pairs whose first radio of the pair in in [0,180[ or  [360,540[
+    for i in list(range(n_angles_pi)) + list(range(n_angles_2pi, n_angles_2pi + n_angles_pi)):
+        merits = []
+        energies = []
+        z_transl = []
+
+        if (not np.isnan(diag.zpix_transl[i])) and (not np.isnan(diag.zpix_transl[i + n_angles_pi])):
+            radio1 = diag.radios[i]
+            radio2 = diag.radios[i + n_angles_pi]
+
+            weight1 = diag.weights[i]
+            weight2 = diag.weights[i + n_angles_pi]
+
+            for overlap in overlap_list:
+                if overlap <= ovsd_sx:
+                    my_overlap = overlap
+                    my_radio1 = radio1
+                    my_radio2 = radio2
+                    my_weight1 = weight1
+                    my_weight2 = weight2
+                else:
+                    my_overlap = ovsd_sx - (overlap - ovsd_sx)
+                    my_radio1 = np.fliplr(radio1)
+                    my_radio2 = np.fliplr(radio2)
+                    my_weight1 = np.fliplr(weight1)
+                    my_weight2 = np.fliplr(weight2)
+
+                radio_common_left = np.fliplr(my_radio1[:, ovsd_sx - my_overlap :])[
+                    :, : -(args.ovs * args.high_pass * 2)
+                ]
+                radio_common_right = my_radio2[:, ovsd_sx - my_overlap : -(args.ovs * args.high_pass * 2)]
+                diff_common = radio_common_right - radio_common_left
+
+                weight_common_left = np.fliplr(my_weight1[:, ovsd_sx - my_overlap :])[
+                    :, : -(args.ovs * args.high_pass * 2)
+                ]
+                weight_common_right = my_weight2[:, ovsd_sx - my_overlap : -(args.ovs * args.high_pass * 2)]
+                weight_common = weight_common_right * weight_common_left
+
+                if args.use_l1_norm:
+                    merits.append(abs(diff_common * weight_common).astype("d").sum())
+                    energies.append(abs(weight_common).astype("d").sum())
+                else:
+                    merits.append((diff_common * diff_common * weight_common).astype("d").sum())
+                    energies.append(
+                        (
+                            (radio_common_left * radio_common_left + radio_common_right * radio_common_right)
+                            * weight_common
+                        )
+                        .astype("d")
+                        .sum()
+                    )
+        else:
+            merits = [0] * (len(overlap_list))
+            energies = [0] * (len(overlap_list))
+
+        z_transl = 0.5 * (diag.zpix_transl[i] + diag.zpix_transl[i + n_angles_pi])
+        all_z_transls.append(z_transl)
+
+        all_merits.append(merits)
+        all_energies.append(energies)
+
+    return all_merits, all_energies, all_z_transls
+
+
+def find_best_interpolating_line(args):
+    (all_z_transl, index_overlap_list_a, index_overlap_list_b, all_energies, all_res) = args
+
+    z_a = np.nanmin(all_z_transl)
+    z_b = np.nanmax(all_z_transl)
+
+    best_error = np.nan
+    best_off_pair = None
+
+    for index_ovlp_a in index_overlap_list_a:
+        for index_ovlp_b in index_overlap_list_b:
+            index_ovlps = np.interp(all_z_transl, [z_a, z_b], [index_ovlp_a, index_ovlp_b])
+            indexes = (np.arange(all_energies.shape[0]))[~np.isnan(index_ovlps)].astype("i")
+
+            index_ovlps = index_ovlps[~np.isnan(index_ovlps)]
+            index_ovlps = np.round_(index_ovlps).astype("i")
+
+            diff_enes = all_res[(indexes, index_ovlps)]
+            orig_enes = all_energies[(indexes, index_ovlps)]
+
+            error = (diff_enes / (orig_enes + 1.0e-30)).astype("d").sum()
+
+            if error == error:
+                if not (error > best_error):
+                    best_error = error
+                    best_error_pair = index_ovlp_a, index_ovlp_b
+
+    return best_error, best_error_pair
+
+
+def main(user_args=None):
+    """Find the cor as a function f z translation and write an hdf5 which contains interpolable tables.
+    This file can be used subsequently with the correct-rot utility.
+    """
+
+    if user_args is None:
+        user_args = sys.argv[1:]
+
+    args = DictToObj(
+        parse_params_values(
+            DiagToRotConfig,
+            parser_description=main.__doc__,
+            program_version="nabu " + version,
+            user_args=user_args,
+        )
+    )
+
+    if args.near is None:
+        if args.original_scan is None:
+            raise ValueError(
+                "the parameter near was not provided but the original_scan parameter was not provided either"
+            )
+        if args.entry_name is None:
+            raise ValueError(
+                "the parameter near was not provided but the entry_name parameter  for the original scan was not provided either"
+            )
+
+        scan = NXtomo()
+        scan.load(file_path=args.original_scan, data_path=args.entry_name)
+        args.near = scan.instrument.detector.estimated_cor_from_motor
+    else:
+        pass
+
+    args.ovs = 4
+
+    diag_url = os.path.join("/", args.entry_name, "reconstruction/configuration/diagnostics")
+
+    diag_keys = []
+    with h5py.File(args.diag_file, "r") as f:
+        diag_keys = list(f[diag_url].keys())
+        diag_keys = [diag_keys[i] for i in np.argsort(list(map(int, diag_keys)))]
+
+    all_merits = []
+    all_energies = []
+    all_z_transls = []
+
+    arguments_for_multiprocessing = []
+
+    for i_key, my_key in enumerate(diag_keys):
+        diag = DictToObj(h5todict(args.diag_file, os.path.join(diag_url, my_key)))
+
+        args.original_shape = diag.radios[0].shape
+        args.zpix_mm = diag.pixel_size_mm
+
+        transform_images(diag, args)
+
+        if i_key == 0:
+            orig_sy, ovsd_sx = diag.radios[0].shape  # already transformed here, ovsd_sx is expanded
+            args.ovsd_sx = ovsd_sx
+
+            overlap_min = max(4, ovsd_sx - 2 * args.ovs * (args.near + args.near_width))
+            overlap_max = min(2 * ovsd_sx - 4, ovsd_sx - 2 * args.ovs * (args.near - args.near_width))
+
+            overlap_list = list(range(int(overlap_min), int(overlap_max) + 1))
+
+            if overlap_min > overlap_max:
+                message = f""" There is no safe search range in find_cor once the margins corresponding to the high_pass filter are discarded.
+                May be the near value (which is the offset respect to the center of the image) is too big, or too negative,
+                in short too close to the borders.
+                """
+                raise ValueError(message)
+
+        arguments_for_multiprocessing.append((diag, overlap_list, args))
+
+    ncpus = get_available_threads()
+    with Pool(processes=ncpus) as pool:
+        result_list = pool.map(total_merit_list, arguments_for_multiprocessing)
+
+    for merits, energies, z_transls in result_list:
+        all_z_transls.extend(z_transls)
+        all_merits.extend(merits)
+        all_energies.append(energies)
+
+    n_pairings_with_data = 0
+    for en, me in zip(all_merits, all_energies):
+        if np.any(me):
+            n_pairings_with_data += 1
+
+    if args.linear_interpolation:
+        if n_pairings_with_data < 2:
+            message = f""" The diagnostics collection has probably been run over a too thin section of the scan
+            or you scan does not allow to form pairs of theta, theta+360. I only found {n_pairings_with_data} pairings
+            and this is not enough to do correlation + interpolation between sections
+            """
+            raise RuntimeError(message)
+    else:
+        if n_pairings_with_data < 1:
+            message = f""" The diagnostics collection has probably been run over a too thin section of the scan
+            or you scan does not allow to form pairs of theta, theta+360. I only found {n_pairings_with_data}
+            pairings
+            """
+            raise RuntimeError(message)
+
+    # all_merits, all_energies, all_z_transls = zip( result_list )
+
+    # merits, energies, z_transls = total_merit_list(diag, overlap_list, args)
+
+    #     all_z_transls.extend(z_transls)
+    #     all_merits.extend(merits)
+    #     all_energies.append(energies)
+
+    all_merits = np.array(all_merits)
+    all_energies = np.array(all_energies)
+
+    all_merits.shape = -1, len(overlap_list)
+    all_energies.shape = -1, len(overlap_list)
+
+    if args.linear_interpolation:
+        do_linear_interpolation(args, overlap_list, all_merits, all_energies, all_z_transls)
+    else:
+        do_height_by_height(args, overlap_list, all_merits, all_energies, all_z_transls)
+
+    return 0
+
+
+def do_height_by_height(args, overlap_list, all_diff, all_energies, all_z_transl):
+    # now we find the best cor for each chunk, or nan if no overlap is found
+    z_a = np.min(all_z_transl)
+    z_b = np.max(all_z_transl)
+
+    grouped_diff = {}
+    grouped_energy = {}
+
+    for diff, energy, z in zip(all_diff, all_energies, all_z_transl):
+        found = z
+        for key in grouped_diff.keys():
+            if abs(key - z) < 2.0:  # these are in pixel units
+                found = key
+                break
+        grouped_diff[found] = grouped_diff.get(found, np.zeros([len(overlap_list)], "f")) + diff
+        grouped_energy[found] = grouped_energy.get(found, np.zeros([len(overlap_list)], "f")) + energy
+
+    z_list = list(grouped_energy.keys())
+    z_list.sort()
+
+    cor_list = []
+    for z in z_list:
+        diff = grouped_diff[z]
+        energy = grouped_energy[z]
+
+        best_error = np.nan
+        best_off = None
+
+        if not np.isnan(z):
+            for i_ovlp in range(len(overlap_list)):
+                error = diff[i_ovlp] / (energy[i_ovlp] + 1.0e-30)
+
+                if not (error > best_error):
+                    best_error = error
+                    best_off = i_ovlp
+
+            if best_off is not None:
+                offset = (args.ovsd_sx - overlap_list[best_off]) / args.ovs / 2
+                sy, sx = args.original_shape
+                cor_abs = (sx - 1) / 2 + offset
+                cor_list.append(cor_abs)
+            else:
+                cor_list.append(np.nan)
+        else:
+            # no overlap  was available for that z
+            cor_list.append(np.nan)
+
+    with h5py.File(args.cor_file, "w") as f:
+        my_mask = ~np.isnan(np.array(cor_list))
+        f["cor"] = np.array(cor_list)[my_mask]
+        f["z_pix"] = np.array(z_list)[my_mask]
+        f["z_m"] = (np.array(z_list)[my_mask]) * args.zpix_mm * 1.0e-3
+
+
+def do_linear_interpolation(args, overlap_list, all_res, all_energies, all_z_transl):
+    # now we consider all the linear regressions of the offset with z_transl
+
+    ncpus = get_available_threads()
+
+    index_overlap_list = np.arange(len(overlap_list)).astype("i")
+    arguments_list = [
+        (all_z_transl, piece, index_overlap_list, all_energies, all_res)
+        for piece in np.array_split(index_overlap_list, ncpus)
+    ]
+
+    with Pool(processes=ncpus) as pool:
+        result_list = pool.map(find_best_interpolating_line, arguments_list)
+
+    error_list = [tok[0] for tok in result_list]
+    best_pos = np.argmin(error_list)
+    best_error, best_error_pair = result_list[best_pos]
+
+    # find the interpolated line
+    i_ovlp_a, i_ovlp_b = best_error_pair
+    offset_a = (args.ovsd_sx - overlap_list[i_ovlp_a]) / args.ovs / 2
+    offset_b = (args.ovsd_sx - overlap_list[i_ovlp_b]) / args.ovs / 2
+
+    sy, sx = args.original_shape
+
+    cor_abs_a = (sx - 1) / 2 + offset_a
+    cor_abs_b = (sx - 1) / 2 + offset_b
+
+    z_a = np.nanmin(all_z_transl)
+    z_b = np.nanmax(all_z_transl)
+
+    with h5py.File(args.cor_file, "w") as f:
+        f["cor"] = np.array([cor_abs_a, cor_abs_b])
+        f["z_pix"] = np.array([z_a, z_b])
+        f["z_m"] = np.array([z_a, z_b]) * args.zpix_mm * 1.0e-3

build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/nabu/app/double_flatfield.py

@@ -0,0 +1,143 @@
+import numpy as np
+from ..preproc.double_flatfield import DoubleFlatField
+from ..preproc.flatfield import FlatFieldDataUrls
+from ..io.reader import ChunkReader
+from ..io.writer import NXProcessWriter
+from ..resources.dataset_analyzer import analyze_dataset
+from ..resources.nxflatfield import update_dataset_info_flats_darks
+from ..resources.logger import Logger, LoggerOrPrint
+from .cli_configs import DFFConfig
+from .utils import parse_params_values
+
+
+class DoubleFlatFieldChunks:
+    def __init__(
+        self, dataset_path, output_file, chunk_size=100, sigma=None, do_flatfield=True, h5_entry=None, logger=None
+    ):
+        self.logger = LoggerOrPrint(logger)
+        self.dataset_info = analyze_dataset(dataset_path, extra_options={"hdf5_entry": h5_entry}, logger=logger)
+        self.do_flatfield = bool(do_flatfield)
+        if self.do_flatfield:
+            update_dataset_info_flats_darks(self.dataset_info, flatfield_mode="force-compute")
+        self.output_file = output_file
+        self.sigma = sigma if sigma is not None and abs(sigma) > 1e-5 else None
+        self._init_reader(chunk_size)
+        self._init_flatfield((None, None, 0, self.chunk_size))
+        self._init_dff()
+
+    def _init_reader(self, chunk_size, start_idx=0):
+        self.chunk_size = min(chunk_size, self.dataset_info.radio_dims[-1])
+        self.reader = ChunkReader(
+            self.dataset_info.projections,
+            sub_region=(None, None, start_idx, start_idx + self.chunk_size),
+            convert_float=True,
+        )
+        self.projections = self.reader.files_data
+
+    def _init_flatfield(self, subregion):
+        if not self.do_flatfield:
+            return
+        self.flatfield = FlatFieldDataUrls(
+            (self.dataset_info.n_angles, self.chunk_size, self.dataset_info.radio_dims[0]),
+            self.dataset_info.flats,
+            self.dataset_info.darks,
+            sorted(self.dataset_info.projections.keys()),
+            sub_region=subregion,
+        )
+
+    def _apply_flatfield(self):
+        if self.do_flatfield:
+            self.flatfield.normalize_radios(self.projections)
+
+    def _set_reader_subregion(self, subregion):
+        self.reader._set_subregion(subregion)
+        self.reader._init_reader()
+        self.reader._loaded = False
+
+    def _init_dff(self):
+        self.double_flatfield = DoubleFlatField(
+            self.projections.shape,
+            input_is_mlog=False,
+            output_is_mlog=False,
+            average_is_on_log=self.sigma is not None,
+            sigma_filter=self.sigma,
+        )
+
+    def _get_config(self):
+        conf = {
+            "dataset": self.dataset_info.location,
+            "entry": self.dataset_info.hdf5_entry or None,
+            "dff_sigma": self.sigma,
+            "do_flatfield": self.do_flatfield,
+        }
+        return conf
+
+    def compute_double_flatfield(self):
+        """
+        Compute the double flatfield for the current dataset.
+        """
+        n_z = self.dataset_info.radio_dims[-1]
+        chunk_size = self.chunk_size
+        n_steps = n_z // chunk_size
+        extra_step = bool(n_z % chunk_size)
+        res = np.zeros(self.dataset_info.radio_dims[::-1])
+        for i in range(n_steps):
+            self.logger.debug("Computing DFF batch %d/%d" % (i + 1, n_steps + int(extra_step)))
+            subregion = (None, None, i * chunk_size, (i + 1) * chunk_size)
+            self._set_reader_subregion(subregion)
+            self._init_flatfield(subregion)
+            self.reader.load_files()
+            self._apply_flatfield()
+            dff = self.double_flatfield.compute_double_flatfield(self.projections, recompute=True)
+            res[subregion[-2] : subregion[-1]] = dff[:]
+        # Need to initialize objects with a different shape
+        if extra_step:
+            curr_idx = (i + 1) * self.chunk_size
+            self.logger.debug("Computing DFF batch %d/%d" % (i + 2, n_steps + int(extra_step)))
+            self._init_reader(n_z - curr_idx, start_idx=curr_idx)
+            self._init_flatfield(self.reader.sub_region)
+            self._init_dff()
+            self.reader.load_files()
+            self._apply_flatfield()
+            dff = self.double_flatfield.compute_double_flatfield(self.projections, recompute=True)
+            res[curr_idx:] = dff[:]
+        return res
+
+    def write_double_flatfield(self, arr):
+        """
+        Write the double flatfield image to a file
+        """
+        writer = NXProcessWriter(
+            self.output_file,
+            entry=self.dataset_info.hdf5_entry or "entry",
+            filemode="a",
+            overwrite=True,
+        )
+        writer.write(arr, "double_flatfield", config=self._get_config())
+        self.logger.info("Wrote %s" % writer.fname)
+
+
+def dff_cli():
+    args = parse_params_values(
+        DFFConfig, parser_description="A command-line utility for computing the double flatfield of a dataset."
+    )
+    logger = Logger("nabu_double_flatfield", level=args["loglevel"], logfile="nabu_double_flatfield.log")
+
+    output_file = args["output"]
+
+    dff = DoubleFlatFieldChunks(
+        args["dataset"],
+        output_file,
+        chunk_size=args["chunk_size"],
+        sigma=args["sigma"],
+        do_flatfield=bool(args["flatfield"]),
+        h5_entry=args["entry"] or None,
+        logger=logger,
+    )
+    dff_image = dff.compute_double_flatfield()
+    dff.write_double_flatfield(dff_image)
+    return 0
+
+
+if __name__ == "__main__":
+    dff_cli()