nxtomomill 2.1.0.dev2__tar.gz → 2.1.0.dev3__tar.gz

{nxtomomill-2.1.0.dev2/nxtomomill.egg-info → nxtomomill-2.1.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nxtomomill
-Version: 2.1.0.dev2
+Version: 2.1.0.dev3
 Summary: applications and library to convert raw format to NXtomo format
 Author-email: Henri Payno <henri.payno@esrf.fr>, Pierre Paleo <pierre.paleo@esrf.fr>, Pierre-Olivier Autran <pierre-olivier.autran@esrf.fr>, Jérôme Lesaint <jerome.lesaint@esrf.fr>, Alessandro Mirone <mirone@esrf.fr>
 License: 
@@ -55,13 +55,14 @@ License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: h5py>=3.0
 Requires-Dist: silx>=2.0
-Requires-Dist: nxtomo>=3.0.0dev0
+Requires-Dist: nxtomo>=3.1.0dev0
 Requires-Dist: pint
 Requires-Dist: packaging
 Requires-Dist: tomoscan[full]>=2.2.0a4
 Requires-Dist: tqdm
 Requires-Dist: pydantic
 Requires-Dist: eval_type_backport; python_version < "3.10"
+Requires-Dist: platformdirs
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: python-gitlab; extra == "test"

{nxtomomill-2.1.0.dev2 → nxtomomill-2.1.0.dev3}/nxtomomill/app/h52nx.py

@@ -11,7 +11,6 @@ For a complete tutorial you can have a look at: :ref:`Tomoh52nx`
 
 import argparse
 import logging
-import os
 
 from tqdm import tqdm
 
@@ -206,9 +205,10 @@ def main(argv):
         help="Y pixel size key to read",
     )
     parser.add_argument(
-        "--read-mode",
+        "--open_mode_for_reading",
+        "--open-mode-for-reading",
         default="robust",
-        help="Options when reading bliss file. Valid values are 'locking', 'swmr' and 'safe'. 'robust' is the default h5py read mode.",
+        help="Options when reading bliss file. Valid values are 'robust' or 'safe'. 'robust' is the default h5py read mode.",
     )
     # scan titles
     parser.add_argument(
@@ -301,21 +301,23 @@ def main(argv):
         help="file containing the full configuration to convert from h5 "
         "bliss to nexus",
     )
-
+    parser.add_argument(
+        "--use-abs-path-for-vds",
+        "--abs-path-for-vds",
+        "--use_abs_path_for_vds",
+        "--use-abs-path-for-vds",
+        default=False,
+        action="store_true",
+        help="Generated NXtomo file contains relative path by default. "
+        "Sometime relative link cannot be solved (GPFS vs NFS). "
+        "In this case you might be forced to use absolute path.",
+    )
     options = parser.parse_args(argv[1:])
     if options.request_input:
         callback_det_sel = _ask_for_selecting_detector
     else:
         callback_det_sel = None
 
-    # define reading mode
-    read_mode = os.getenv("NXTOMOMILL_HDF5_READ_MODE", "robust")
-    if options.read_mode not in ("robust", "safe", "swmr"):
-        raise ValueError(
-            f"Invalid read mode: {read_mode}. Valid values are 'robust', 'safe' and 'swmr'"
-        )
-    os.environ["NXTOMOMILL_HDF5_READ_MODE"] = options.read_mode
-
     try:
         configuration_handler = TomoHDF5ConfigHandler(options)
     except Exception:

{nxtomomill-2.1.0.dev2 → nxtomomill-2.1.0.dev3}/nxtomomill/app/patch_nx.py

@@ -253,7 +253,9 @@ def main(argv):
             "No url provided for dark or flats or frame type to "
             "modify. Nothing to be done."
         )
-    elif not utils.is_nx_tomo_entry(file_path=options.file_path, entry=options.entry):
+    elif not utils.is_nx_tomo_entry(
+        file_path=options.file_path, entry=options.entry, open_mode_for_reading="safe"
+    ):
         _logger.error(
             f"{options.entry}@{options.file_path} is not recognized as a valid NXTomo entry."
         )
@@ -292,4 +294,5 @@ def main(argv):
                 flats_end=flat_end_url,
                 embed_data=True,
                 logger=_logger,
+                open_mode_for_reading="robust",
             )

{nxtomomill-2.1.0.dev2 → nxtomomill-2.1.0.dev3}/nxtomomill/app/split_nxfile.py

@@ -47,7 +47,17 @@ def main(argv):
         help="Make all NXtomo free of any external link. As a result this will duplicate data",
         nargs="?",
     )
-
+    parser.add_argument(
+        "--use_abs_path_for_vds",
+        "--use-abs-path-for-vds",
+        "--abs-path-for-vds",
+        "--abs_path_for_vds",
+        default=False,
+        action="store_true",
+        help="Generated NXtomo file contains relative path by default. "
+        "Sometime relative link cannot be solved (GPFS vs NFS). "
+        "In this case you might be forced to use absolute path.",
+    )
     options = parser.parse_args(argv[1:])
     output_file_pattern = options.output_file_pattern
     if "{input_file_name}" in output_file_pattern:
@@ -61,6 +71,7 @@ def main(argv):
         input_file=options.input_file,
         output_file_pattern=output_file_pattern,
         overwrite=options.overwrite,
+        force_abs_path=options.use_abs_path_for_vds,
     )
 
 
@@ -69,11 +80,13 @@ def split(
     output_file_pattern: str,
     overwrite: bool = False,
     duplicate_data: bool = False,
+    force_abs_path: bool = False,
 ) -> tuple:
     """
     :param input_file: path to the file to be splitted
     :param output_file_pattern: pattern of the file to create. Must contain either `{entry_name}` or `{index}`
     :param overwrite: can we overwrite output file if alread exists
+    :param force_abs_path: should the generated NXtomo contains absolute path (True) or relative path (False, default)
 
     :return: tuple of identifier of all the NXtomo generated
     """
@@ -128,7 +141,10 @@ def split(
                 if dirname and not os.path.exists(dirname):
                     os.makedirs(dirname)
                 nx_tomo.save(
-                    file_path=output_file, data_path=entry, overwrite=overwrite
+                    file_path=output_file,
+                    data_path=entry,
+                    overwrite=overwrite,
+                    use_abs_path_for_vds=force_abs_path,
                 )
                 result.append(NXtomoScan(output_file, entry))
 

{nxtomomill-2.1.0.dev2 → nxtomomill-2.1.0.dev3}/nxtomomill/app/z_concatenate_scans.py

@@ -209,7 +209,9 @@ def main(argv):
         if dummy_frame is None:
             if len(nxt.instrument.detector.data) > 0:
                 # create empty frame to give it to the 'invalid_nxt' NXtomo later
-                with DatasetReader(nxt.instrument.detector.data[0]) as raw_data:
+                with DatasetReader(
+                    nxt.instrument.detector.data[0], open_mode_for_reading="safe"
+                ) as raw_data:
                     data_type = raw_data.dtype
                 # FIXME: avoid keeping some file open. not clear why this is needed
                 raw_data = None

{nxtomomill-2.1.0.dev2 → nxtomomill-2.1.0.dev3}/nxtomomill/converter/dxfile/dxfileconverter.py

@@ -143,7 +143,7 @@ class _DxFileToNxConverter(BaseConverter):
             output_file = os.path.join(
                 os.path.dirname(input_file),
                 os.path.basename(input_file_basename)
-                + self._configuration.file_extension,
+                + self._configuration.file_extension.value,
             )
         else:
             output_file = os.path.abspath(self.output_file)
@@ -329,7 +329,9 @@ class _DxFileToNxConverter(BaseConverter):
         self._copy_count_time()
 
         # start time
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             if "file_creation_datetime" in input_entry:
                 with HDF5File(self.output_file, mode="a") as h5f:
                     root_grp = h5f.require_group(self.output_entry)
@@ -365,7 +367,9 @@ class _DxFileToNxConverter(BaseConverter):
                         f"Failed to get 'count_time' / 'exposure_period'. reason is {e}"
                     )
 
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             if "measurement" in input_entry:
                 measurement_grp = input_entry["measurement"]
                 if "detector" in measurement_grp:
@@ -469,7 +473,9 @@ class _DxFileToNxConverter(BaseConverter):
         n_proj = 0
         # handle darks
         try:
-            with DatasetReader(self._data_darks_url) as dark_dataset:
+            with DatasetReader(
+                self._data_darks_url, open_mode_for_reading="safe"
+            ) as dark_dataset:
                 n_dark = dark_dataset.shape[0]
             # FIXME: avoid keeping some file open. not clear why this is needed
             dark_dataset = None
@@ -491,7 +497,9 @@ class _DxFileToNxConverter(BaseConverter):
 
         # handle flats
         try:
-            with DatasetReader(self._data_flats_url) as flat_dataset:
+            with DatasetReader(
+                self._data_flats_url, open_mode_for_reading="safe"
+            ) as flat_dataset:
                 n_flat = flat_dataset.shape[0]
             # FIXME: avoid keeping some file open. not clear why this is needed
             flat_dataset = None
@@ -513,7 +521,9 @@ class _DxFileToNxConverter(BaseConverter):
 
         # handle projections
         try:
-            with DatasetReader(self._data_proj_url) as proj_dataset:
+            with DatasetReader(
+                self._data_proj_url, open_mode_for_reading="safe"
+            ) as proj_dataset:
                 n_proj = proj_dataset.shape[0]
             # FIXME: avoid keeping some file open. not clear why this is needed
             proj_dataset = None
@@ -554,7 +564,9 @@ class _DxFileToNxConverter(BaseConverter):
             self._path_nxtomo_attrs(root_grp)
 
     def _read_sample_detector_distance(self):
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             path = "/".join(
                 ("measurement", "instrument", "sample", "setup", "detector_distance")
             )
@@ -568,7 +580,9 @@ class _DxFileToNxConverter(BaseConverter):
             return distance * unit
 
     def _read_energy(self):
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             path = "/".join(("measurement", "instrument", "source", "energy"))
             dataset = input_entry[path]
             energy = float(h5py_read_dataset(dataset))
@@ -580,7 +594,9 @@ class _DxFileToNxConverter(BaseConverter):
             return energy * unit
 
     def _read_x_pixel_size(self):
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             path = "/".join(
                 ("measurement", "instrument", "detector", "actual_pixel_size_x")
             )
@@ -594,7 +610,9 @@ class _DxFileToNxConverter(BaseConverter):
             return pixel_size * unit
 
     def _read_y_pixel_size(self):
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             path = "/".join(
                 ("measurement", "instrument", "detector", "actual_pixel_size_y")
             )

nxtomomill-2.1.0.dev3/nxtomomill/converter/edf/_Blosc2Jp2kWrapper.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+import numpy
+from h5py import Dataset
+
+try:
+    import blosc2
+    import blosc2_grok
+except ImportError:
+    _has_blosc2 = False
+else:
+    _has_blosc2 = True
+
+
+class Blosc2Jp2kWrapper:
+    """Wrapper to apply JP2K conversion to a dataset"""
+
+    def __init__(self, dataset: Dataset, compression_params: dict):
+        self.dataset: Dataset = dataset
+        self.compression_params: dict = self.get_default_compression_params()
+        self.compression_params.update(compression_params)
+
+        blosc2_grok.set_params_defaults(
+            # codec=blosc2.Codec.GROK,
+            cod_format=blosc2_grok.GrkFileFmt.GRK_FMT_JP2,
+            num_threads=self.compression_params["nthreads"],
+            quality_mode="rates",
+            quality_layers=numpy.array(
+                [self.compression_params["cratio"]],
+                dtype=numpy.float64,
+            ),
+            verbose=True,
+        )
+
+    def __setitem__(self, name, value):
+        z, _, _ = name
+        Z, Y, X = self.dataset.shape
+
+        # Verify data type compatibility
+        if value.dtype not in [numpy.float32, numpy.float64, numpy.uint16]:
+            raise ValueError(f"Unsupported dtype: {value.dtype}")
+
+        b2im = blosc2.asarray(
+            value.reshape(1, Y, X),
+            chunks=(1, Y, X),
+            blocks=(1, Y, X),
+            cparams=self.compression_params,
+        )
+        cframe = b2im.schunk.to_cframe()
+        self.dataset.id.write_direct_chunk((z, 0, 0), cframe, filter_mask=0)
+
+    @classmethod
+    def get_default_compression_params(cls) -> dict:
+        if _has_blosc2:
+            return {
+                "codec": blosc2.Codec.GROK,
+                "filters": [],
+                "splitmode": blosc2.SplitMode.NEVER_SPLIT,
+                "nthreads": 4,
+                "cratio": 10,
+                "verbose": True,
+            }
+        else:
+            return {}

nxtomomill-2.1.0.dev3/nxtomomill/converter/hdf5/acquisition/_bliss_activeconfig.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import h5py
+from silx.io.utils import h5py_read_dataset
+
+__all__ = [
+    "BlissActiveConfig",
+]
+
+
+class BlissActiveConfig:
+    """
+    hold the bliss active config. Can retrieve the 'tomo_detector' that contains information about the optic
+    """
+
+    def __init__(self) -> None:
+        self._tomo_detector: None | tuple[str, ...] = None
+
+    @property
+    def tomo_detector(self) -> tuple[str, ...] | None:
+        return self._tomo_detector
+
+    @staticmethod
+    def from_hdf5_group(group: h5py.Group):
+        """
+        Create an instance of 'BlissActiveConfig' from a group (containing the bliss active configuration).
+        """
+        if not isinstance(group, h5py.Group):
+            raise TypeError
+
+        bliss_config = BlissActiveConfig()
+        if "tomo_detector" in group:
+            bliss_config._tomo_detector = h5py_read_dataset(group["tomo_detector"])
+        return bliss_config

{nxtomomill-2.1.0.dev2 → nxtomomill-2.1.0.dev3}/nxtomomill/converter/hdf5/acquisition/baseacquisition.py

@@ -64,6 +64,7 @@ class BaseAcquisition:
     )
 
     _INSTRUMENT_NAME_PATH = (
+        "instrument/name",
         "technique/saving/beamline",
         "instrument/title",
     )
@@ -98,14 +99,10 @@ class BaseAcquisition:
 
     _SOURCE_NAME = ("instrument/machine/name",)
 
-    _BEAMLINE_NAME = (
-        "instrument/name",
-        "instrument/title",
-    )
-
     _SOURCE_TYPE = ("instrument/machine/type",)
 
     _FRAME_FLIP_PATHS = (
+        "instrument/{active_config_tomo_detector}/data_axes",  # note: data_axes are stored in the optic and not in the camera name.
         "technique/detector/{detector_name}/flipping",
         "technique/detector/flipping",
     )
@@ -203,7 +200,7 @@ class BaseAcquisition:
             if self.configuration.single_file or not divide_sub_files:
                 en_output_file = self.configuration.output_file
             else:
-                ext = file_extension_ or self.configuration.file_extension
+                ext = file_extension_ or self.configuration.file_extension.value
                 file_name = (
                     os.path.splitext(output_file_basename)[0]
                     + "_"
@@ -284,7 +281,10 @@ class BaseAcquisition:
                         and self._copy_frames[url.path()]
                     ):
                         new_url = embed_url(url, output_file=output_file)
-                        n_vs, _, _ = from_data_url_to_virtual_source(new_url)
+                        n_vs, _, _ = from_data_url_to_virtual_source(
+                            new_url,
+                            open_mode_for_reading=self.configuration.open_mode_for_reading,
+                        )
                         new_vs.append(n_vs)
                     else:
                         new_vs.append(vs)
@@ -294,7 +294,11 @@ class BaseAcquisition:
             if input_file_path is not None:
                 nx_tomo.bliss_original_files = (abspath(input_file_path),)
             # save data
-            nx_tomo.save(file_path=output_file, data_path=data_path)
+            nx_tomo.save(
+                file_path=output_file,
+                data_path=data_path,
+                use_abs_path_for_vds=self.configuration.use_abs_path_for_vds,
+            )
             # check rotation angle
             if nx_tomo.sample.rotation_angle is not None:
                 unique_angles = numpy.unique(
@@ -331,7 +335,10 @@ class BaseAcquisition:
         raise NotImplementedError("Base class")
 
     def read_entry(self):
-        return EntryReader(self._root_url)
+        return EntryReader(
+            self._root_url,
+            open_mode_for_reading=self.configuration.open_mode_for_reading,
+        )
 
     def is_different_sequence(self, entry):
         """
@@ -346,7 +353,9 @@ class BaseAcquisition:
             return False
 
         def read_sample_node(url: DataUrl) -> str | None:
-            with EntryReader(url=url) as entry:
+            with EntryReader(
+                url=url, open_mode_for_reading=self.configuration.open_mode_for_reading
+            ) as entry:
                 sample_node = self._get_sample_node(entry)
                 if sample_node is None:
                     return None
@@ -536,7 +545,10 @@ class BaseAcquisition:
             _logger.warning("no root url. Unable to read rotation motor")
             return None
         else:
-            with EntryReader(self._root_url) as entry:
+            with EntryReader(
+                self._root_url,
+                open_mode_for_reading=self.configuration.open_mode_for_reading,
+            ) as entry:
                 for motor_path in self._TECHNIQUE_MOTOR_PATHS:
                     if motor_path in entry:
                         try:
@@ -809,7 +821,10 @@ class BaseAcquisition:
             _logger.warning("no root url. Unable to read detector roi")
             return None
         else:
-            with EntryReader(self._root_url) as entry:
+            with EntryReader(
+                self._root_url,
+                open_mode_for_reading=self.configuration.open_mode_for_reading,
+            ) as entry:
                 for roi_path in self._DETECTOR_ROI:
                     if roi_path in entry:
                         try:

{nxtomomill-2.1.0.dev2 → nxtomomill-2.1.0.dev3}/nxtomomill/converter/hdf5/acquisition/multitomo.py

@@ -5,7 +5,9 @@ module to define a multi-tomo acquistion (also know as pco-tomo)
 
 from __future__ import annotations
 
+import numpy
 import logging
+from datetime import timezone
 
 from silx.io.url import DataUrl
 from silx.utils.proxy import docstring
@@ -129,21 +131,27 @@ class MultiTomoAcquisition(StandardAcquisition):
                     )
 
     def get_nb_loop(self, url) -> int | None:
-        with EntryReader(url) as entry:
+        with EntryReader(
+            url, open_mode_for_reading=self.configuration.open_mode_for_reading
+        ) as entry:
             for path in self._NB_LOOP_PATH:
                 if path in entry:
                     return entry[path][()]
         return None
 
     def get_nb_tomo(self, url) -> int | None:
-        with EntryReader(url) as entry:
+        with EntryReader(
+            url, open_mode_for_reading=self.configuration.open_mode_for_reading
+        ) as entry:
             for path in self._NB_TOMO_PATH:
                 if path in entry:
                     return entry[path][()]
         return None
 
     def get_nb_turns(self, url) -> int | None:
-        with EntryReader(url) as entry:
+        with EntryReader(
+            url, open_mode_for_reading=self.configuration.open_mode_for_reading
+        ) as entry:
             for path in self._NB_TURNS_PATH:
                 if path in entry:
                     return entry[path][()]
@@ -167,7 +175,12 @@ class MultiTomoAcquisition(StandardAcquisition):
         return the multitomo version according to the provider version (aka bliss)
         """
         with EntryReader(
-            url=DataUrl(file_path=url.file_path(), data_path="/", scheme=url.scheme())
+            url=DataUrl(
+                file_path=url.file_path(),
+                data_path="/",
+                scheme=url.scheme(),
+            ),
+            open_mode_for_reading=self.configuration.open_mode_for_reading,
         ) as entry:
             if "creator_version" in entry.attrs:
                 creator_version = entry.attrs["creator_version"]
@@ -263,13 +276,43 @@ class MultiTomoAcquisition(StandardAcquisition):
             elif len(projections_slices) == 0:
                 raise ValueError("No projection found")
             else:
-                results.extend(
-                    splitter.split(
-                        projections_slices[0],
-                        nb_part=int(self._nb_turns),
-                        tomo_n=self._get_tomo_n(),
+                tomo_n = self._get_tomo_n()
+                try:
+                    start_times, end_times = self._estimate_start_and_stop_times(
+                        proj_start_index=projections_slices[0].start,
+                        proj_stop_index=projections_slices[0].stop,
+                        time_stamps=sorted(self._frames_timestamp),
+                        n_proj_per_nxtomo=tomo_n,
+                    )
+                except IndexError as e:
+                    _logger.error(
+                        f"Fail to estimate sub-NXtomo start and stop time. Error is {e}."
                     )
+                    start_times = end_times = None
+
+                sub_nxtomos = splitter.split(
+                    projections_slices[0],
+                    nb_part=int(self._nb_turns),
+                    tomo_n=tomo_n,
                 )
+                # update sub-NXtomo `start_time` and `end_time`.
+                if start_times is not None and end_times is not None:
+                    if len(start_times) != len(sub_nxtomos):
+                        _logger.error(
+                            "Incoherent number of 'start_time' and generated sub-NXtomo. Won't modify start time."
+                        )
+                    else:
+                        for start_time, end_time, sub_nxtomo in zip(
+                            start_times, end_times, sub_nxtomos
+                        ):
+                            sub_nxtomo.start_time = start_time.item().replace(
+                                tzinfo=timezone.utc
+                            )
+                            sub_nxtomo.end_time = end_time.item().replace(
+                                tzinfo=timezone.utc
+                            )
+
+                results.extend(sub_nxtomos)
 
         if check_tomo_n:
             # if angle offset provided there is an hight probably this will mess up with the tomo_n, which is expected...
@@ -374,6 +417,40 @@ class MultiTomoAcquisition(StandardAcquisition):
 
         return tuple(results)
 
+    @staticmethod
+    def _estimate_start_and_stop_times(
+        proj_start_index: int,
+        proj_stop_index: int,
+        time_stamps: tuple[numpy.datetime64],
+        n_proj_per_nxtomo: None | int,
+    ) -> tuple[tuple[numpy.datetime64], tuple[numpy.datetime64]] | None:
+        """
+        Estimate the starts time for sub NXtomos. For this we read the timestamp of the first projection.
+        We canno use the dark / flats that could be duplicated or the original start time which is the start of the bliss sequence.
+
+        .. warning: this could create a discrepancy for other scans that are using the bliss-tomo start time.
+        """
+        if n_proj_per_nxtomo is None:
+            _logger.error(
+                "Unable to know the number of projection expected per nxtomo. Unable to estimate start time."
+            )
+            return None
+        first_projection_indices = numpy.arange(
+            proj_start_index,
+            proj_stop_index,
+            step=n_proj_per_nxtomo,
+            dtype=numpy.uint32,
+        )
+        last_projection_indices = numpy.arange(
+            proj_start_index + n_proj_per_nxtomo - 1,
+            proj_stop_index + 1,  # +1 because stop index is not included in the range
+            step=n_proj_per_nxtomo,
+            dtype=numpy.uint32,
+        )
+        return tuple([time_stamps[i] for i in first_projection_indices]), tuple(
+            [time_stamps[i] for i in last_projection_indices]
+        )
+
     @staticmethod
     def _get_projections_slices(nx_tomo: NXtomo) -> tuple:
         """Return a tuple of slices for each group of contiguous projections"""