nabu 2022.3.0a1__py3-none-any.whl → 2023.1.0a2__py3-none-any.whl

nabu/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2022.3.0-a1"
+__version__ = "2023.1.0-alpha2"
 __nabu_modules__ = [
     "app",
     "cuda",

nabu/app/bootstrap.py

@@ -3,6 +3,7 @@ from glob import glob
 from ..utils import get_folder_path
 from ..pipeline.config import generate_nabu_configfile, parse_nabu_config_file
 from ..pipeline.fullfield.nabu_config import nabu_config as default_fullfield_config
+from ..pipeline.helical.nabu_config import nabu_config as helical_fullfield_config
 from .utils import parse_params_values
 from .cli_configs import BootstrapConfig
 
@@ -46,9 +47,14 @@ def bootstrap():
             exit(1)
         prefilled_values["dataset"]["location"] = user_dataset
 
+    if args["helical"]:
+        my_config = helical_fullfield_config
+    else:
+        my_config = default_fullfield_config
+
     generate_nabu_configfile(
         args["output"],
-        default_fullfield_config,
+        my_config,
         comments=not (no_comments),
         options_level=opts_level,
         prefilled_values=prefilled_values,

nabu/app/cast_volume.py

@@ -27,8 +27,9 @@ from nabu.pipeline.params import files_formats
 _logger = logging.getLogger(__name__)
 
 
-def main(argv):
-
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
     _volume_url_helps = "\n".join(
         [
             f"- {(volume.__name__).ljust(15)}: {volume.example_defined_from_str_identifier()}"
@@ -100,6 +101,11 @@ def main(argv):
 
     if os.path.exists(options.input_volume):
         volumes = guess_volumes(options.input_volume)
+
+        def is_not_histogram(vol_identifier):
+            return not (hasattr(vol_identifier, "data_path") and vol_identifier.data_path.endswith("histogram"))
+
+        volumes = tuple(filter(is_not_histogram, volumes))
         if len(volumes) == 0:
             _logger.error(f"no valid volume found in {options.input_volume}")
             exit(1)

nabu/app/cli_configs.py

@@ -37,6 +37,7 @@ BootstrapConfig = {
         "help": "Use a template configuration file. Available are: id19_pag, id16_holo, id16_ctf. You can also define your own templates via the NABU_TEMPLATES_PATH environment variable.",
         "default": "",
     },
+    "helical": {"help": "Prepare configuration file for helical", "default": 0, "required": False, "type": int},
 }
 
 # Default configuration for "zsplit" command
@@ -352,3 +353,71 @@ ShrinkConfig = {
         "type": int,
     },
 }
+
+CompositeCorConfig = {
+    "--filename_template": {
+        "required": True,
+        "help": """The filename template. It can optionally contain a segment equal to "X"*ndigits which will be replaced by the stage number if several stages are requested by the user""",
+    },
+    "--entry_name": {
+        "required": False,
+        "help": "Optional. The entry_name. It defaults to entry0000",
+        "default": "entry0000",
+    },
+    "--num_of_stages": {
+        "type": int,
+        "required": False,
+        "help": "Optional. How many stages. Example: from 0 to 43 -> --num_of_stages  44. It is optional. ",
+    },
+    "--oversampling": {
+        "type": int,
+        "default": 4,
+        "required": False,
+        "help": "Oversampling in the research of the axis position. Defaults to 4 ",
+    },
+    "--n_subsampling_y": {
+        "type": int,
+        "default": 10,
+        "required": False,
+        "help": "How many lines we are going to take from each radio. Defaults to 10.",
+    },
+    "--theta_interval": {
+        "type": float,
+        "default": 5,
+        "required": False,
+        "help": "Angular step for composing the image. Default to 5",
+    },
+    "--first_stage": {"type": int, "default": None, "required": False, "help": "Optional. The first stage.  "},
+    "--output_file": {
+        "type": str,
+        "required": False,
+        "help": "Optional. Where the list of cors will be written. Default is the filename postixed with cors.txt",
+    },
+    "--cor_options": {
+        "type": str,
+        "help": """the cor_options string used by Nabu. Example 
+        --cor_options "side='near'; near_pos = 300.0;  near_width = 20.0"
+        """,
+        "required": True,
+    },
+}
+
+CreateDistortionMapHorizontallyMatchedFromPolyConfig = {
+    "--nz": {"type": int, "help": "vertical dimension of the detector", "required": True},
+    "--nx": {"type": int, "help": "horizontal dimension of the detector", "required": True},
+    "--center_z": {"type": float, "help": "vertical position of the optical center", "required": True},
+    "--center_x": {"type": float, "help": "horizontal position of the optical center", "required": True},
+    "--c4": {"type": float, "help": "order 4 coefficient", "required": True},
+    "--c2": {"type": float, "help": "order 2 coefficient", "required": True},
+    "--target_file": {"type": str, "help": "The map output filename", "required": True},
+    "--axis_pos": {
+        "type": float,
+        "default": None,
+        "help": "Optional argument. If given it will be corrected for use with the produced map. The value is printed, or given as return argument if the utility is used from a script",
+        "required": False,
+    },
+    "--loglevel": {
+        "help": "Logging level. Can be 'debug', 'info', 'warning', 'error'. Default is 'info'.",
+        "default": "info",
+    },
+}

nabu/app/composite_cor.py

@@ -0,0 +1,97 @@
+import logging
+import os
+import sys
+import numpy as np
+import re
+
+from nabu.resources.dataset_analyzer import HDF5DatasetAnalyzer
+from nabu.pipeline.estimators import CompositeCOREstimator
+from nabu.resources.nxflatfield import update_dataset_info_flats_darks
+from .. import version
+from .cli_configs import CompositeCorConfig
+from .utils import parse_params_values
+from ..utils import DictToObj
+
+
+def main(user_args=None):
+    "Application to extract with the composite cor finder the center of rotation for a scan or a series of scans"
+
+    if user_args is None:
+        user_args = sys.argv[1:]
+
+    args = DictToObj(
+        parse_params_values(
+            CompositeCorConfig,
+            parser_description=main.__doc__,
+            program_version="nabu " + version,
+            user_args=user_args,
+        )
+    )
+
+    if len(os.path.dirname(args.filename_template)) == 0:
+        # To make sure that other utility routines can succesfully deal with path within the current directory
+        args.filename_template = os.path.join(".", args.filename_template)
+
+    if args.first_stage is not None:
+        if args.num_of_stages is None:
+            args.num_of_stages = 1
+        # if the first_stage parameter has been given then
+        # we are using numbers to form the names of the files.
+        # The filename must containe a XX..X substring which will be replaced
+        pattern = re.compile("[X]+")
+        ps = pattern.findall(args.filename_template)
+        if len(ps) == 0:
+            message = f""" You have specified the "first_stage" parameter, with an integer.
+            Therefore the "filename_template" parameter is expected to containe a XX..X subsection
+            but none was found in the passed parameter which is {args.filename_template}
+            """
+            raise ValueError(message)
+        ls = list(map(len, ps))
+        idx = np.argmax(ls)
+
+        args.filename_template = args.filename_template.replace(ps[idx], "{i_stage:" + "0" + str(ls[idx]) + "d}")
+
+    if args.num_of_stages is None:
+        # this way it works also in the simple case where
+        # only the filename is provided together with the cor options
+        num_of_stages = 1
+        first_stage = 0
+    else:
+        num_of_stages = args.num_of_stages
+        first_stage = args.first_stage
+    cor_list = []
+    for iz in range(first_stage, first_stage + num_of_stages):
+        if args.num_of_stages is not None:
+            nexus_name = args.filename_template.format(i_stage=iz)
+        else:
+            nexus_name = args.filename_template
+
+        dataset_info = HDF5DatasetAnalyzer(nexus_name)
+
+        update_dataset_info_flats_darks(dataset_info, flatfield_mode=1)
+
+        cor_finder = CompositeCOREstimator(
+            dataset_info,
+            oversampling=args.oversampling,
+            theta_interval=args.theta_interval,
+            n_subsampling_y=args.n_subsampling_y,
+            take_log=True,
+            spike_threshold=0.04,
+            cor_options=args.cor_options,
+        )
+
+        cor_position = cor_finder.find_cor()
+
+        cor_list.append(cor_position)
+
+    cor_list = np.array(cor_list).T
+
+    if args.output_file is not None:
+        output_name = args.output_file
+    else:
+        output_name = os.path.splitext(args.filename_template)[0] + "_cors.txt"
+
+    np.savetxt(
+        output_name,
+        cor_list,
+    )

nabu/app/create_distortion_map_from_poly.py

@@ -0,0 +1,118 @@
+from .. import version
+import numpy as np
+import h5py
+import argparse
+import sys
+from .cli_configs import CreateDistortionMapHorizontallyMatchedFromPolyConfig
+from .utils import parse_params_values
+from ..utils import DictToObj
+from ..resources.logger import Logger, LoggerOrPrint
+
+
+def horizontal_match(user_args=None):
+    """This application builds  two arrays. Let us call them map_x and map_z. Both are 2D arrays with shape given by (nz, nx).
+    These maps are meant to be used to generate a corrected detector image, using them to obtain  the pixel (i,j) of the corrected
+    image by interpolating the raw data at position ( map_z(i,j), map_x(i,j) ).
+
+    This map is determined by a user given  polynomial  P(rs) in the radial variable  rs = sqrt( (z-center_z)**2 + (x-center_x)**2 ) / (nx/2)
+    where center_z and center_x give the center around which the deformation is centered.
+
+    The perfect  position (zp,xp) , that would be observed on a perfect detector, of a photon observed at pixel (z,x) of the distorted detector is:
+
+       (zp, xp) = (center_z,  center_x) +  P(rs) *  ( z - center_z  , x - center_x )
+
+    The polynomial is given by P(rs) = rs *(1 + c2 * rs**2 + c4 * rs**4)
+
+    The map is rescaled and reshifted so that a perfect match is realised at the borders of a horizontal line passing by the center. This ensures coerence
+    with the procedure of pixel size calibration which is performed moving a needle horizontally and reading the motor positions at the extreme positions.
+
+    The maps are written in the target file, creating it as hdf5 file,  in the datasets
+
+        "/coords_source_x"
+        "/coords_source_z"
+
+    The URLs of these two maps can be used for the detector correction of type "map_xz"
+    in the nabu configuration file as in this example
+
+         [dataset]
+         ...
+         detector_distortion_correction = map_xz
+         detector_distortion_correction_options = map_x="silx:./map_coordinates.h5?path=/coords_source_x" ;  map_z="silx:./map_coordinates.h5?path=/coords_source_z"
+
+    """
+
+    if user_args is None:
+        user_args = sys.argv[1:]
+
+    args = DictToObj(
+        parse_params_values(
+            CreateDistortionMapHorizontallyMatchedFromPolyConfig,
+            parser_description=horizontal_match.__doc__,
+            program_version="nabu " + version,
+            user_args=user_args,
+        )
+    )
+    logger = Logger("horizontal_match", level=args.loglevel, logfile="horizontal_match.log")
+
+    nz, nx = args.nz, args.nx
+    center_x, center_z = (args.center_x, args.center_z)
+
+    c4 = args.c4
+    c2 = args.c2
+
+    polynomial = np.poly1d([c4, 0, c2, 0, 1, 0.0])
+    # change of variable
+    cofv = np.poly1d([1.0 / (nx / 2), 0])
+    polynomial = nx / 2 * polynomial(cofv)
+
+    left_border = 0 - center_x
+    right_border = nx - 1 - center_x
+
+    def get_rescaling_shift(left_border, right_border, polynomial):
+        dl = polynomial(left_border)
+        dr = polynomial(right_border)
+
+        rescaling = (dr - dl) / (right_border - left_border)
+        shift = -left_border * rescaling + dl
+        return rescaling, shift
+
+    final_grid_rescaling, final_grid_shift = get_rescaling_shift(left_border, right_border, polynomial)
+
+    coords_z, coords_x = np.indices([nz, nx])
+
+    coords_z = ((coords_z - center_z) * final_grid_rescaling).astype("d")
+    coords_x = ((coords_x - center_x) * final_grid_rescaling + final_grid_shift).astype("d")
+
+    distances_goal = np.sqrt(coords_z * coords_z + coords_x * coords_x)
+
+    distances_unknown = distances_goal
+
+    pp_deriv = polynomial.deriv()
+
+    # iteratively finding the positions to interpolated at by newton method
+    for i in range(10):
+        errors = polynomial(distances_unknown) - distances_goal
+        derivative = pp_deriv(distances_unknown)
+        distances_unknown = distances_unknown - errors / derivative
+
+    distances_data_sources = distances_unknown
+
+    # avoid 0/0
+    distances_data_sources[distances_goal < 1] = 1
+    distances_goal[distances_goal < 1] = 1
+
+    coords_source_z = coords_z * distances_data_sources / distances_goal + center_z
+    coords_source_x = coords_x * distances_data_sources / distances_goal + center_x
+
+    with h5py.File(args.target_file, "w") as f:
+        f["coords_source_x"] = coords_source_x
+        f["coords_source_z"] = coords_source_z
+
+    if args.axis_pos is not None:
+        coord_axis = args.axis_pos - center_x
+        new_pos = (polynomial(coord_axis) - final_grid_shift) / final_grid_rescaling + center_x
+        logger.info("New axis position at %e it was previously  %e " % (new_pos, args.axis_pos))
+
+        return new_pos
+    else:
+        return None

nabu/app/nx_z_splitter.py

@@ -89,7 +89,7 @@ class NXZSplitter:
         masks = [(z_transl == z) for z in different_z]
         for i_z, mask in enumerate(masks):
             fname_curr_z = path.join(
-                self.output_dir, path.splitext(path.basename(self.fname))[0] + str("_%04d" % i_z) + self._ext
+                self.output_dir, path.splitext(path.basename(self.fname))[0] + str("_%06d" % i_z) + self._ext
             )
             self.logger.info("Creating %s" % fname_curr_z)
             copy_file(self.fname, fname_curr_z)

nabu/app/prepare_weights_double.py

@@ -5,12 +5,13 @@ from scipy.special import erf
 import sys
 import os
 from scipy.ndimage import gaussian_filter
+from tomoscan.esrf.scan.hdf5scan import ImageKey, HDF5TomoScan
 
 
 def main():
     """auxiliary program that can be called to create  default input detector profiles, for nabu helical,
     concerning the weights of the pixels and the "double flat" renormalisation denominator.
-    The result is an hdf5 file that can be used as a "processes" nabu file and is used by nabu-helical.
+    The result is an hdf5 file that can be used as a "processes_file" in the nabu configuration and is used by nabu-helical.
     In particulars cases the user may have fancy masks and correction map and will provide its own processes file,
     and will not need this.
 
@@ -27,32 +28,35 @@ def main():
     Then the resulting file can be used as processes file in the configuration file of nabu-helical
 
     """
-    if len(sys.argv) not in [3, 4]:
+    if len(sys.argv) not in [3, 4, 5]:
         message = f""" Usage:
-            {os.path.basename(sys.argv[0])}   nexus_file_name entry_name [target file name]
+            {os.path.basename(sys.argv[0])}   nexus_file_name entry_name [target_file name [transition_width]]
         """
         print(message)
         sys.exit(1)
 
     file_name = sys.argv[1]
     entry_name = sys.argv[2]
-    if len(sys.argv) != 4:
-        process_file_name = "double.h5"
-    else:
+    process_file_name = "double.h5"
+    transition_width = 50.0
+
+    if len(sys.argv) in [4, 5]:
         process_file_name = sys.argv[3]
+        if len(sys.argv) in [5]:
+            transition_width = float(sys.argv[4])
 
-    f = h5py.File(file_name, "r")
+    scan = HDF5TomoScan(file_name, entry_name)
+    scan_flats, metadata_flats = scan.load_reduced_flats(return_info=True)
 
-    im_keys = f[entry_name]["data/image_key"][()]
-    which_slices = list(np.where(np.equal(im_keys, 1))[0])
-    mappe = np.array([f[entry_name]["data/data"][i] for i in which_slices])
-    mappe = mappe.sum(0)
-    f.close()
+    mappe = 0
+    for key, flat in scan_flats.items():
+        mappe += flat
+    mappe = mappe / len(list(scan_flats.keys()))
 
-    create_heli_maps(mappe, process_file_name, entry_name)
+    create_heli_maps(mappe, process_file_name, entry_name, transition_width)
 
 
-def create_heli_maps(profile, process_file_name, entry_name):
+def create_heli_maps(profile, process_file_name, entry_name, transition_width):
     profile = profile / profile.max()
     profile = profile.astype("f")
 
@@ -64,7 +68,6 @@ def create_heli_maps(profile, process_file_name, entry_name):
         fd = h5py.File(process_file_name, "w")
 
     def f(L, m, w):
-
         x = np.arange(L)
 
         d = (x - L + m).astype("f")
@@ -77,6 +80,8 @@ def create_heli_maps(profile, process_file_name, entry_name):
 
     border = f(profile.shape[1], 20, 13.33)
 
+    border_v = f(profile.shape[0], int(round(transition_width / 2)), transition_width / 4)
+
     path_weights = entry_name + "/weights_field/results/data"
     path_double = entry_name + "/double_flatfield/results/data"
 
@@ -85,5 +90,5 @@ def create_heli_maps(profile, process_file_name, entry_name):
     if path_double in fd:
         del fd[path_double]
 
-    fd[path_weights] = profile * border
+    fd[path_weights] = (profile * border) * border_v[:, None]
     fd[path_double] = np.ones_like(profile)

nabu/app/reconstruct_helical.py

@@ -64,7 +64,6 @@ def main_helical():
         Nx, Ny = proc.dataset_info.radio_dims
 
         if proc.nabu_config["reconstruction"]["auto_size"]:
-
             if 2 * rot_center > Nx:
                 w = int(round(2 * rot_center))
             else:

nabu/app/utils.py

@@ -1,16 +1,21 @@
 from argparse import ArgumentParser
 
 
-def parse_params_values(Params, parser_description=None, program_version=None):
+def parse_params_values(Params, parser_description=None, program_version=None, user_args=None):
     parser = ArgumentParser(description=parser_description)
     for param_name, vals in Params.items():
-        optional = not (vals.pop("mandatory", False))
-        if optional:
-            param_name = "--" + param_name
+        if param_name[0] != "-":
+            # It would be better to use "required" and not to pop it.
+            # required is an accepted keyword for argparse
+            optional = not (vals.pop("mandatory", False))
+            if optional:
+                param_name = "--" + param_name
         parser.add_argument(param_name, **vals)
     if program_version is not None:
         parser.add_argument("--version", "-V", action="version", version=program_version)
-    args = parser.parse_args()
+
+    args = parser.parse_args(args=user_args)
+
     args_dict = args.__dict__
     return args_dict
 

nabu/cuda/processing.py

@@ -7,6 +7,7 @@ if __has_pycuda__:
 
     dev_attrs = cuda.device_attribute
 
+
 # NB: we must detach from a context before creating another context
 class CudaProcessing:
     def __init__(self, device_id=None, ctx=None, stream=None, cleanup_at_exit=True):

nabu/cuda/tests/test_padding.py

@@ -18,6 +18,7 @@ scenarios_legacy = [
     },
 ]
 
+
 # parametrize with fixture and "params=" will launch a new class for each scenario.
 # the attributes set to "cls" will remain for all the tests done in this class
 # with the current scenario.

nabu/cuda/utils.py

@@ -45,6 +45,7 @@ def get_cuda_context(device_id=None, cleanup_at_exit=True):
     # Unlike Context.make_context(), the newly-created context is not made current.
     context = cuda.Device(device_id).retain_primary_context()
     context.push()
+
     # Register a clean-up function at exit
     def _finish_up(context):
         if context is not None:

nabu/distributed/utils.py

@@ -0,0 +1,57 @@
+from os import getpid
+from datetime import datetime
+from socket import gethostname
+
+
+# Default formatter
+FORMATTER = "[{hostname}/{pid}/{worker}] {day}/{month}/{year} {hour}:{min}:{sec} {message}"
+
+
+def format_message(message, worker_name=None, replacements=None):
+    """
+    Format a message with additional information.
+    The formating is defined by `nabu.distributed.utils.FORMATTER`.
+
+    Parameters
+    -----------
+    message: str
+        Message to log.
+    worker_name: str, optional
+        Name of the current worker.
+    replacements: dict, optional
+        Dictionary of additional patterns that should be replaced in the current
+        formatter.
+    """
+    now = datetime.now()
+    # poor man's templating engine
+    formatters = {
+        "hostname": gethostname(),
+        "pid": getpid(),
+        "worker": "",
+        "day": now.strftime("%d"),
+        "month": now.strftime("%m"),
+        "year": now.strftime("%Y"),
+        "hour": now.strftime("%H"),
+        "min": now.strftime("%M"),
+        "sec": now.strftime("%S"),
+        "message": message
+    }
+    if worker_name is not None:
+        formatters["worker"] = worker_name
+    FMT = FORMATTER.replace("{", "").replace("}", "")
+    for fmt_key, fmt_val in formatters.items():
+        FMT = FMT.replace(fmt_key, str(fmt_val))
+    if replacements is not None:
+        for what, replacement in replacements.items():
+            FMT = FMT.replace(what, replacement)
+    return FMT
+
+
+def log(message):
+    """
+    Log a message using the current formater.
+    """
+    print(format_message(message))
+
+
+