nabu 2024.1.0rc3__py3-none-any.whl → 2024.1.1__py3-none-any.whl

build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/nabu/app/reduce_dark_flat.py

@@ -0,0 +1,181 @@
+import sys
+import logging
+import argparse
+from typing import Optional
+
+from nabu.app.cli_configs import ReduceDarkFlatConfig
+from .utils import parse_params_values
+from .. import version
+
+from tomoscan.framereducer.method import ReduceMethod
+from tomoscan.scanbase import TomoScanBase
+from tomoscan.esrf.scan.edfscan import EDFTomoScan
+from tomoscan.factory import Factory
+from silx.io.url import DataUrl
+
+
+def _create_data_urls(output_file: Optional[str], output_data_path: Optional[str], name: str):
+    """
+    util function to compute reduced Data and metadata url(s)
+    This only handle the case of hdf5 outputs
+    """
+    assert name in ("flats", "darks"), f"name is '{name}'"
+
+    def get_data_paths(data_path: Optional[str]) -> tuple:
+        """return (data_path, metadta_path)"""
+        if data_path is None:
+            return "{entry}/" + name + "/{index}", "{entry}/" + name
+        elif not data_path.endswith("/{index}"):
+            # we are not expecting useds to provide the index but only upstream part
+            return data_path + "/{index}", data_path
+        else:
+            raise RuntimeError(
+                "unhandled use case (/index provided) and don;t know where to set the data and the metadata"
+            )
+
+    data_path, metadata_path = get_data_paths(output_data_path)
+
+    output_file = output_file or "{scan_prefix}_" + f"{name}.hdf5"
+    data_urls = [
+        DataUrl(
+            file_path=output_file,
+            data_path=data_path,
+            scheme="silx",
+        ),
+    ]
+
+    metadata_urls = [
+        DataUrl(
+            file_path=output_file,
+            data_path=metadata_path,
+            scheme="silx",
+        ),
+    ]
+    return data_urls, metadata_urls
+
+
+def reduce_dark_flat(
+    scan: TomoScanBase,
+    dark_method: ReduceMethod,
+    flat_method: ReduceMethod,
+    overwrite: bool = False,
+    output_reduced_darks_file: Optional[str] = None,
+    output_reduced_darks_data_path: Optional[str] = None,
+    output_reduced_flats_file: Optional[str] = None,
+    output_reduced_flats_data_path: Optional[str] = None,
+) -> int:
+    """
+    calculation of the darks / flats calling tomoscan utils function
+    """
+    dark_method = ReduceMethod.from_value(dark_method) if dark_method is not None else None
+    flat_method = ReduceMethod.from_value(flat_method) if flat_method is not None else None
+
+    # 1. define url where to save the file
+    ## 1.1 for darks
+    if dark_method is None:
+        reduced_darks_data_urls = ()
+        reduced_darks_metadata_urls = ()
+    elif output_reduced_darks_file is None and output_reduced_darks_data_path is None:
+        # if no settings provided then take the default path (the idea is also to be more robust to future modifications)
+        reduced_darks_data_urls = scan.REDUCED_DARKS_DATAURLS
+        reduced_darks_metadata_urls = scan.REDUCED_DARKS_METADATAURLS
+    elif isinstance(scan, EDFTomoScan):
+        # simplification of the equation
+        raise ValueError("reduce-dark-flat can only compute create dark-flats at default location for edf")
+    else:
+        reduced_darks_data_urls, reduced_darks_metadata_urls = _create_data_urls(
+            output_file=output_reduced_darks_file,
+            output_data_path=output_reduced_darks_data_path,
+            name="darks",
+        )
+    ## 1.2 for flats
+    if flat_method is None:
+        reduced_flats_data_urls = ()
+        reduced_flats_metadata_urls = ()
+    elif output_reduced_flats_file is None and output_reduced_flats_data_path is None:
+        # if no settings provided then take the default path (the idea is also to be more robust to future modifications)
+        reduced_flats_data_urls = scan.REDUCED_FLATS_DATAURLS
+        reduced_flats_metadata_urls = scan.REDUCED_FLATS_METADATAURLS
+    elif isinstance(scan, EDFTomoScan):
+        # simplification of the equation
+        raise ValueError("reduce-dark-flat can only compute create dark-flats at default location for edf")
+    else:
+        reduced_flats_data_urls, reduced_flats_metadata_urls = _create_data_urls(
+            output_file=output_reduced_flats_file,
+            output_data_path=output_reduced_flats_data_path,
+            name="flats",
+        )
+
+    # 2. compute and save darks / flats
+
+    success = True
+    ## 2.1 handle dark
+    if dark_method is not None:
+        try:
+            reduced_darks, darks_metadata = scan.compute_reduced_darks(
+                reduced_method=dark_method,
+                overwrite=overwrite,
+                return_info=True,
+            )
+        except Exception as e:
+            print(f"failed to create reduced darks. Error is {e}")
+            success = False
+        else:
+            scan.save_reduced_darks(
+                darks=reduced_darks,
+                darks_infos=darks_metadata,
+                output_urls=reduced_darks_data_urls,
+                metadata_output_urls=reduced_darks_metadata_urls,
+                overwrite=overwrite,
+            )
+
+    ## 2.2 handle flats
+    if flat_method is not None:
+        try:
+            reduced_flats, flats_metadata = scan.compute_reduced_flats(
+                reduced_method=flat_method,
+                overwrite=overwrite,
+                return_info=True,
+            )
+        except Exception as e:
+            print(f"failed to create reduced flats. Error is {e}")
+            success = False
+        else:
+            scan.save_reduced_flats(
+                flats=reduced_flats,
+                flats_infos=flats_metadata,
+                output_urls=reduced_flats_data_urls,
+                metadata_output_urls=reduced_flats_metadata_urls,
+                overwrite=overwrite,
+            )
+
+    return success
+
+
+def main(argv=None):
+    """
+    Compute reduce dark(s) and flat(s) of a dataset
+    """
+    if argv is None:
+        argv = sys.argv[1:]
+
+    args = parse_params_values(
+        ReduceDarkFlatConfig,
+        parser_description=main.__doc__,
+        program_version="nabu " + version,
+        user_args=argv,
+    )
+
+    scan = Factory.create_scan_object(args["dataset"], entry=args["entry"])
+    exit(
+        reduce_dark_flat(
+            scan=scan,
+            dark_method=args["dark_method"],
+            flat_method=args["flat_method"],
+            overwrite=args["overwrite"],
+            output_reduced_darks_file=args["output_reduced_darks_file"],
+            output_reduced_darks_data_path=args["output_reduced_darks_data_path"],
+            output_reduced_flats_file=args["output_reduced_flats_file"],
+            output_reduced_flats_data_path=args["output_reduced_flats_data_path"],
+        )
+    )

build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/nabu/app/rotate.py

@@ -0,0 +1,162 @@
+import posixpath
+from os import path
+from math import ceil
+from shutil import copy
+from multiprocessing import cpu_count
+from multiprocessing.pool import ThreadPool
+import numpy as np
+from tomoscan.io import HDF5File
+from tomoscan.esrf.scan.nxtomoscan import NXtomoScan
+from ..io.utils import get_first_hdf5_entry
+from ..processing.rotation import Rotation
+from ..resources.logger import Logger, LoggerOrPrint
+from ..pipeline.config_validators import optional_tuple_of_floats_validator, boolean_validator
+from ..processing.rotation_cuda import CudaRotation, __has_pycuda__
+from .utils import parse_params_values
+from .cli_configs import RotateRadiosConfig
+
+
+class HDF5ImagesStackRotation:
+    def __init__(
+        self,
+        input_file,
+        output_file,
+        angle,
+        center=None,
+        entry=None,
+        logger=None,
+        batch_size=100,
+        use_cuda=True,
+        use_multiprocessing=True,
+    ):
+        self.logger = LoggerOrPrint(logger)
+        self.use_cuda = use_cuda & __has_pycuda__
+        self.batch_size = batch_size
+        self.use_multiprocessing = use_multiprocessing
+        self._browse_dataset(input_file, entry)
+        self._get_rotation(angle, center)
+        self._init_output_dataset(output_file)
+
+    def _browse_dataset(self, input_file, entry):
+        self.input_file = input_file
+        if entry is None or entry == "":
+            entry = get_first_hdf5_entry(input_file)
+        self.entry = entry
+        self.dataset_info = NXtomoScan(input_file, entry=entry)
+
+    def _get_rotation(self, angle, center):
+        if self.use_cuda:
+            self.logger.info("Using Cuda rotation")
+            rot_cls = CudaRotation
+        else:
+            self.logger.info("Using skimage rotation")
+            rot_cls = Rotation
+            if self.use_multiprocessing:
+                self.thread_pool = ThreadPool(processes=cpu_count() - 2)
+                self.logger.info("Using multiprocessing with %d cores" % self.thread_pool._processes)
+
+        self.rotation = rot_cls((self.dataset_info.dim_2, self.dataset_info.dim_1), angle, center=center, mode="edge")
+
+    def _init_output_dataset(self, output_file):
+        self.output_file = output_file
+        copy(self.input_file, output_file)
+
+        first_proj_url = self.dataset_info.projections[list(self.dataset_info.projections.keys())[0]]
+        self.data_path = first_proj_url.data_path()
+        dirname, basename = posixpath.split(self.data_path)
+        self._data_path_dirname = dirname
+        self._data_path_basename = basename
+
+    def _rotate_stack_cuda(self, images, output):
+        # pylint: disable=E1136
+        self.rotation.cuda_processing.allocate_array("tmp_images_stack", images.shape)
+        self.rotation.cuda_processing.allocate_array("tmp_images_stack_rot", images.shape)
+        d_in = self.rotation.cuda_processing.get_array("tmp_images_stack")
+        d_out = self.rotation.cuda_processing.get_array("tmp_images_stack_rot")
+        n_imgs = images.shape[0]
+        d_in[:n_imgs].set(images)
+        for j in range(n_imgs):
+            self.rotation.rotate(d_in[j], output=d_out[j])
+        d_out[:n_imgs].get(ary=output[:n_imgs])
+
+    def _rotate_stack(self, images, output):
+        if self.use_cuda:
+            self._rotate_stack_cuda(images, output)
+        elif self.use_multiprocessing:
+            out_tmp = self.thread_pool.map(self.rotation.rotate, images)
+            print(out_tmp[0])
+            output[:] = np.array(out_tmp, dtype="f")  # list -> np array... consumes twice as much memory
+        else:
+            for j in range(images.shape[0]):
+                output[j] = self.rotation.rotate(images[j])
+
+    def rotate_images(self, suffix="_rot"):
+        data_path = self.data_path
+        fid = HDF5File(self.input_file, "r")
+        fid_out = HDF5File(self.output_file, "a")
+
+        try:
+            data_ptr = fid[data_path]
+            n_images = data_ptr.shape[0]
+            data_out_ptr = fid_out[data_path]
+
+            # Delete virtual dataset in output file, create "data_rot" dataset
+            del fid_out[data_path]
+            fid_out[self._data_path_dirname].create_dataset(
+                self._data_path_basename + suffix, shape=data_ptr.shape, dtype=data_ptr.dtype
+            )
+            data_out_ptr = fid_out[data_path + suffix]
+
+            # read by group of images to hide latency
+            group_size = self.batch_size
+            images_rot = np.zeros((group_size, data_ptr.shape[1], data_ptr.shape[2]), dtype="f")
+            n_groups = ceil(n_images / group_size)
+            for i in range(n_groups):
+                self.logger.info("Processing radios group %d/%d" % (i + 1, n_groups))
+                i_min = i * group_size
+                i_max = min((i + 1) * group_size, n_images)
+                images = data_ptr[i_min:i_max, :, :].astype("f")
+                self._rotate_stack(images, images_rot)
+                data_out_ptr[i_min:i_max, :, :] = images_rot[: i_max - i_min, :, :].astype(data_ptr.dtype)
+        finally:
+            fid_out[self._data_path_dirname].move(posixpath.basename(data_path) + suffix, self._data_path_basename)
+            fid_out[data_path].attrs["interpretation"] = "image"
+            fid.close()
+            fid_out.close()
+
+
+def rotate_cli():
+    args = parse_params_values(
+        RotateRadiosConfig,
+        parser_description="A command-line utility for performing a rotation on all the radios of a dataset.",
+    )
+    logger = Logger("nabu_rotate", level=args["loglevel"], logfile="nabu_rotate.log")
+
+    dataset_path = args["dataset"]
+    h5_entry = args["entry"]
+    output_file = args["output"]
+    center = optional_tuple_of_floats_validator("", "", args["center"])  # pylint: disable=E1121
+    use_cuda = boolean_validator("", "", args["use_cuda"])  # pylint: disable=E1121
+    use_multiprocessing = boolean_validator("", "", args["use_multiprocessing"])  # pylint: disable=E1121
+
+    if path.exists(output_file):
+        logger.fatal("Output file %s already exists, not overwriting it" % output_file)
+        exit(1)
+
+    h5rot = HDF5ImagesStackRotation(
+        dataset_path,
+        output_file,
+        args["angle"],
+        center=center,
+        entry=h5_entry,
+        logger=logger,
+        batch_size=args["batchsize"],
+        use_cuda=use_cuda,
+        use_multiprocessing=use_multiprocessing,
+    )
+    h5rot.rotate_images()
+    return 0
+
+
+if __name__ == "__main__":
+    rotate_cli()

build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/nabu/app/shrink_dataset.py

@@ -0,0 +1,98 @@
+import os
+import posixpath
+from multiprocessing.pool import ThreadPool
+import numpy as np
+from silx.io.dictdump import dicttonx, nxtodict
+from ..misc.binning import binning as image_binning
+from ..io.utils import get_first_hdf5_entry
+from ..pipeline.config_validators import optional_tuple_of_floats_validator, optional_positive_integer_validator
+from .cli_configs import ShrinkConfig
+from .utils import parse_params_values
+
+
+def access_nested_dict(dict_, path, default=None):
+    items = [s for s in path.split(posixpath.sep) if len(s) > 0]
+    if len(items) == 1:
+        return dict_.get(items[0], default)
+    if items[0] not in dict_:
+        return default
+    return access_nested_dict(dict_[items[0]], posixpath.sep.join(items[1:]))
+
+
+def set_nested_dict_value(dict_, path, val):
+    dirname, basename = posixpath.split(path)
+    sub_dict = access_nested_dict(dict_, dirname)
+    sub_dict[basename] = val
+
+
+def shrink_dataset(input_file, output_file, binning=None, subsampling=None, entry=None, n_threads=1):
+    entry = entry or get_first_hdf5_entry(input_file)
+    data_dict = nxtodict(input_file, path=entry, dereference_links=False)
+
+    to_subsample = [
+        "control/data",
+        "instrument/detector/count_time",
+        "instrument/detector/data",
+        "instrument/detector/image_key",
+        "instrument/detector/image_key_control",
+        "sample/rotation_angle",
+        "sample/x_translation",
+        "sample/y_translation",
+        "sample/z_translation",
+    ]
+
+    detector_data = access_nested_dict(data_dict, "instrument/detector/data")
+    if detector_data is None:
+        raise ValueError("No data found in %s entry %s" % (input_file, entry))
+
+    if binning is not None:
+
+        def _apply_binning(img_res_tuple):
+            img, res = img_res_tuple
+            res[:] = image_binning(img, binning)
+
+        data_binned = np.zeros(
+            (detector_data.shape[0], detector_data.shape[1] // binning[0], detector_data.shape[2] // binning[1]),
+            detector_data.dtype,
+        )
+        with ThreadPool(n_threads) as tp:
+            tp.map(_apply_binning, zip(detector_data, data_binned))
+        detector_data = data_binned
+        set_nested_dict_value(data_dict, "instrument/detector/data", data_binned)
+
+    if subsampling is not None:
+        for item_path in to_subsample:
+            item_val = access_nested_dict(data_dict, item_path)
+            if item_val is not None:
+                set_nested_dict_value(data_dict, item_path, item_val[::subsampling])
+    dicttonx(data_dict, output_file, h5path=entry)
+
+
+def shrink_cli():
+    args = parse_params_values(ShrinkConfig, parser_description="Shrink a NX dataset")
+
+    if not (os.path.isfile(args["input_file"])):
+        print("No such file: %s" % args["input_file"])
+        exit(1)
+    if os.path.isfile(args["output_file"]):
+        print("Output file %s already exists, not overwriting it" % args["output_file"])
+        exit(1)
+
+    binning = optional_tuple_of_floats_validator("", "binning", args["binning"])  # pylint: disable=E1121
+    if binning is not None:
+        binning = tuple(map(int, binning))
+    subsampling = optional_positive_integer_validator("", "subsampling", args["subsampling"])  # pylint: disable=E1121
+
+    shrink_dataset(
+        args["input_file"],
+        args["output_file"],
+        binning=binning,
+        subsampling=subsampling,
+        entry=args["entry"],
+        n_threads=args["threads"],
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    shrink_cli()

build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/nabu/app/stitching.py

@@ -0,0 +1,85 @@
+import logging
+from nabu.utils import Progress
+from nabu.stitching.slurm_utils import split_stitching_configuration_to_slurm_job
+from .cli_configs import StitchingConfig
+from ..pipeline.config import parse_nabu_config_file
+from nabu.stitching.z_stitching import (
+    z_stitching,
+    StitchingPostProcAggregation,
+)
+from nabu.stitching.config import dict_to_config_obj
+from .utils import parse_params_values
+
+try:
+    from sluurp.executor import submit
+except ImportError:
+    has_sluurp = False
+else:
+    has_sluurp = True
+
+_logger = logging.getLogger(__name__)
+
+
+def main():
+    args = parse_params_values(
+        StitchingConfig,
+        parser_description="Run stitching from a configuration file. Configuration can be obtain from `stitching-config`",
+    )
+    logging.basicConfig(level=args["loglevel"].upper())
+
+    conf_dict = parse_nabu_config_file(args["input-file"], allow_no_value=True)
+
+    stitching_config = dict_to_config_obj(conf_dict)
+    stitching_config.settle_inputs()
+    if args["only_create_master_file"]:
+        # option to ease creation of the master in the following cases:
+        # * user has submitted all the job but has been quicked out of the cluster
+        # * only a few slurm job for some random version (cluster update...) and user want to retriger only those job and process the aggragation only. On those cases no need to redo it all.
+        tomo_objs = []
+        for _, sub_config in split_stitching_configuration_to_slurm_job(stitching_config, yield_configuration=True):
+            tomo_objs.append(sub_config.get_output_object().get_identifier().to_str())
+
+        post_processing = StitchingPostProcAggregation(
+            existing_objs_ids=tomo_objs,
+            stitching_config=stitching_config,
+        )
+        post_processing.process()
+
+    elif stitching_config.slurm_config.partition in (None, ""):
+        # case 1: run locally
+        _logger.info(f"run stitching locally with {stitching_config}")
+
+        progress = Progress("z-stitching")
+        progress.set_name("initialize z-stitching")
+        progress.set_advancement(0)
+        z_stitching(stitching_config, progress=progress)
+    else:
+        if not has_sluurp:
+            raise ImportError(
+                "sluurp not install. Please install it to distribute stitching on slurm (pip install sluurm)"
+            )
+        # case 2: run on slurm
+        # note: to speed up we could do shift research on pre processing and run it only once (if manual of course). Here it will be run for all part
+        _logger.info(f"will distribute stitching")
+
+        futures = {}
+        # 2.1 launch jobs
+        for i_job, (job, sub_config) in enumerate(
+            split_stitching_configuration_to_slurm_job(stitching_config, yield_configuration=True)
+        ):
+            _logger.info(f"submit job nb {i_job}: handles {sub_config.slices}")
+            output_volume = sub_config.get_output_object().get_identifier().to_str()
+            futures[output_volume] = submit(job, timeout=999999)
+
+        # 2.2 wait for future to be done and concatenate the result
+        post_processing = StitchingPostProcAggregation(
+            futures=futures,
+            stitching_config=stitching_config,
+        )
+        post_processing.process()
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/nabu/app/tests/test_reduce_dark_flat.py

@@ -0,0 +1,78 @@
+import os
+import pytest
+from nabu.app.reduce_dark_flat import reduce_dark_flat
+
+#####
+from tomoscan.test.utils import NXtomoMockContext
+
+
+@pytest.fixture(scope="function")
+def hdf5_scan(tmp_path):
+    """simple fixture to create a scan and provide it to another function"""
+    test_dir = tmp_path / "my_hdf5_scan"
+    with NXtomoMockContext(
+        scan_path=str(test_dir),
+        n_proj=10,
+        n_ini_proj=10,
+    ) as scan:
+        yield scan
+
+
+######
+
+
+@pytest.mark.parametrize("dark_method", (None, "first", "mean"))
+@pytest.mark.parametrize("flat_method", (None, "last", "median"))
+def test_reduce_dark_flat_hdf5(tmp_path, hdf5_scan, dark_method, flat_method):  # noqa F811
+    """simply test output - processing is tested at tomoscan side"""
+    # test with default url
+    default_darks_path = os.path.join(hdf5_scan.path, hdf5_scan.get_dataset_basename() + "_darks.hdf5")
+    default_flats_path = os.path.join(hdf5_scan.path, hdf5_scan.get_dataset_basename() + "_flats.hdf5")
+
+    assert not os.path.exists(default_darks_path)
+    assert not os.path.exists(default_flats_path)
+    reduce_dark_flat(
+        scan=hdf5_scan,
+        dark_method=dark_method,
+        flat_method=flat_method,
+    )
+
+    if dark_method is not None:
+        assert os.path.exists(default_darks_path)
+    else:
+        assert not os.path.exists(default_darks_path)
+    if flat_method is not None:
+        assert os.path.exists(default_flats_path)
+    else:
+        assert not os.path.exists(default_flats_path)
+
+    # make sure if already exists and no overwrite fails
+    if dark_method is not None or flat_method is not None:
+        with pytest.raises(KeyError):
+            reduce_dark_flat(
+                scan=hdf5_scan,
+                dark_method=dark_method,
+                flat_method=flat_method,
+                overwrite=False,
+            )
+
+    # test with url provided by the user
+    tuned_darks_path = os.path.join(tmp_path, "new_folder", "darks.hdf5")
+    tuned_flats_path = os.path.join(tmp_path, "new_folder", "flats.hdf5")
+    assert not os.path.exists(tuned_darks_path)
+    assert not os.path.exists(tuned_flats_path)
+    reduce_dark_flat(
+        scan=hdf5_scan,
+        dark_method=dark_method,
+        flat_method=flat_method,
+        output_reduced_darks_file=tuned_darks_path,
+        output_reduced_flats_file=tuned_flats_path,
+    )
+    if dark_method is not None:
+        assert os.path.exists(tuned_darks_path)
+    else:
+        assert not os.path.exists(tuned_darks_path)
+    if flat_method is not None:
+        assert os.path.exists(tuned_flats_path)
+    else:
+        assert not os.path.exists(tuned_flats_path)

build/lib/build/lib/build/lib/build/lib/build/lib/build/lib/nabu/app/utils.py

@@ -0,0 +1,33 @@
+from argparse import ArgumentParser
+
+
+def parse_params_values(Params, parser_description=None, program_version=None, user_args=None):
+    parser = ArgumentParser(description=parser_description)
+    for param_name, vals in Params.items():
+        if param_name[0] != "-":
+            # It would be better to use "required" and not to pop it.
+            # required is an accepted keyword for argparse
+            optional = not (vals.pop("mandatory", False))
+            if optional:
+                param_name = "--" + param_name
+            aliases = vals.pop("aliases", tuple())
+            if optional:
+                aliases = tuple(["--" + alias for alias in aliases])
+        else:
+            aliases = ()
+        parser.add_argument(param_name, *aliases, **vals)
+    if program_version is not None:
+        parser.add_argument("--version", "-V", action="version", version=program_version)
+
+    args = parser.parse_args(args=user_args)
+
+    args_dict = args.__dict__
+    return args_dict
+
+
+def parse_sections(sections):
+    sections = sections.lower()
+    if sections == "all":
+        return None
+    sections = sections.replace(" ", "").split(",")
+    return sections