nxtomomill 2.1.0.dev1__tar.gz → 2.1.0.dev3__tar.gz

{nxtomomill-2.1.0.dev1/nxtomomill.egg-info → nxtomomill-2.1.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nxtomomill
-Version: 2.1.0.dev1
+Version: 2.1.0.dev3
 Summary: applications and library to convert raw format to NXtomo format
 Author-email: Henri Payno <henri.payno@esrf.fr>, Pierre Paleo <pierre.paleo@esrf.fr>, Pierre-Olivier Autran <pierre-olivier.autran@esrf.fr>, Jérôme Lesaint <jerome.lesaint@esrf.fr>, Alessandro Mirone <mirone@esrf.fr>
 License: 
@@ -55,13 +55,14 @@ License-File: LICENSE
 Requires-Dist: numpy
 Requires-Dist: h5py>=3.0
 Requires-Dist: silx>=2.0
-Requires-Dist: nxtomo>=3.0.0dev0
+Requires-Dist: nxtomo>=3.1.0dev0
 Requires-Dist: pint
 Requires-Dist: packaging
 Requires-Dist: tomoscan[full]>=2.2.0a4
 Requires-Dist: tqdm
 Requires-Dist: pydantic
 Requires-Dist: eval_type_backport; python_version < "3.10"
+Requires-Dist: platformdirs
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: python-gitlab; extra == "test"

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev3}/nxtomomill/app/h52nx.py

@@ -11,7 +11,6 @@ For a complete tutorial you can have a look at: :ref:`Tomoh52nx`
 
 import argparse
 import logging
-import os
 
 from tqdm import tqdm
 
@@ -206,9 +205,10 @@ def main(argv):
         help="Y pixel size key to read",
     )
     parser.add_argument(
-        "--read-mode",
-        default="locking",
-        help="Options when reading bliss file. Valid values are 'locking', 'swmr' and 'locking_free'. 'locking' is the default h5py read mode.",
+        "--open_mode_for_reading",
+        "--open-mode-for-reading",
+        default="robust",
+        help="Options when reading bliss file. Valid values are 'robust' or 'safe'. 'robust' is the default h5py read mode.",
     )
     # scan titles
     parser.add_argument(
@@ -284,6 +284,12 @@ def main(argv):
         f"are: {_getPossibleInputParams()}. Should be added at the end of the command line because "
         "will try to cover all text set after this option.",
     )
+    # output entries
+    parser.add_argument(
+        "--output-entries",
+        default=None,
+        help="Provide names of the output entries (HDF5 group path).",
+    )
 
     # config file
     parser.add_argument(
@@ -295,21 +301,23 @@ def main(argv):
         help="file containing the full configuration to convert from h5 "
         "bliss to nexus",
     )
-
+    parser.add_argument(
+        "--use-abs-path-for-vds",
+        "--abs-path-for-vds",
+        "--use_abs_path_for_vds",
+        "--use-abs-path-for-vds",
+        default=False,
+        action="store_true",
+        help="Generated NXtomo file contains relative path by default. "
+        "Sometime relative link cannot be solved (GPFS vs NFS). "
+        "In this case you might be forced to use absolute path.",
+    )
     options = parser.parse_args(argv[1:])
     if options.request_input:
         callback_det_sel = _ask_for_selecting_detector
     else:
         callback_det_sel = None
 
-    # define reading mode
-    read_mode = os.getenv("NXTOMOMILL_HDF5_READ_MODE", "locking")
-    if options.read_mode not in ("locking", "locking_free", "swmr"):
-        raise ValueError(
-            f"Invalid read mode: {read_mode}. Valid values are 'locking', 'locking_free' and 'swmr'"
-        )
-    os.environ["NXTOMOMILL_HDF5_READ_MODE"] = options.read_mode
-
     try:
         configuration_handler = TomoHDF5ConfigHandler(options)
     except Exception:

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev3}/nxtomomill/app/patch_nx.py

@@ -253,7 +253,9 @@ def main(argv):
             "No url provided for dark or flats or frame type to "
             "modify. Nothing to be done."
         )
-    elif not utils.is_nx_tomo_entry(file_path=options.file_path, entry=options.entry):
+    elif not utils.is_nx_tomo_entry(
+        file_path=options.file_path, entry=options.entry, open_mode_for_reading="safe"
+    ):
         _logger.error(
             f"{options.entry}@{options.file_path} is not recognized as a valid NXTomo entry."
         )
@@ -292,4 +294,5 @@ def main(argv):
                 flats_end=flat_end_url,
                 embed_data=True,
                 logger=_logger,
+                open_mode_for_reading="robust",
             )

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev3}/nxtomomill/app/split_nxfile.py

@@ -47,7 +47,17 @@ def main(argv):
         help="Make all NXtomo free of any external link. As a result this will duplicate data",
         nargs="?",
     )
-
+    parser.add_argument(
+        "--use_abs_path_for_vds",
+        "--use-abs-path-for-vds",
+        "--abs-path-for-vds",
+        "--abs_path_for_vds",
+        default=False,
+        action="store_true",
+        help="Generated NXtomo file contains relative path by default. "
+        "Sometime relative link cannot be solved (GPFS vs NFS). "
+        "In this case you might be forced to use absolute path.",
+    )
     options = parser.parse_args(argv[1:])
     output_file_pattern = options.output_file_pattern
     if "{input_file_name}" in output_file_pattern:
@@ -61,6 +71,7 @@ def main(argv):
         input_file=options.input_file,
         output_file_pattern=output_file_pattern,
         overwrite=options.overwrite,
+        force_abs_path=options.use_abs_path_for_vds,
     )
 
 
@@ -69,11 +80,13 @@ def split(
     output_file_pattern: str,
     overwrite: bool = False,
     duplicate_data: bool = False,
+    force_abs_path: bool = False,
 ) -> tuple:
     """
     :param input_file: path to the file to be splitted
     :param output_file_pattern: pattern of the file to create. Must contain either `{entry_name}` or `{index}`
     :param overwrite: can we overwrite output file if alread exists
+    :param force_abs_path: should the generated NXtomo contains absolute path (True) or relative path (False, default)
 
     :return: tuple of identifier of all the NXtomo generated
     """
@@ -128,7 +141,10 @@ def split(
                 if dirname and not os.path.exists(dirname):
                     os.makedirs(dirname)
                 nx_tomo.save(
-                    file_path=output_file, data_path=entry, overwrite=overwrite
+                    file_path=output_file,
+                    data_path=entry,
+                    overwrite=overwrite,
+                    use_abs_path_for_vds=force_abs_path,
                 )
                 result.append(NXtomoScan(output_file, entry))
 

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev3}/nxtomomill/app/z_concatenate_scans.py

@@ -209,7 +209,9 @@ def main(argv):
         if dummy_frame is None:
             if len(nxt.instrument.detector.data) > 0:
                 # create empty frame to give it to the 'invalid_nxt' NXtomo later
-                with DatasetReader(nxt.instrument.detector.data[0]) as raw_data:
+                with DatasetReader(
+                    nxt.instrument.detector.data[0], open_mode_for_reading="safe"
+                ) as raw_data:
                     data_type = raw_data.dtype
                 # FIXME: avoid keeping some file open. not clear why this is needed
                 raw_data = None

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev3}/nxtomomill/converter/dxfile/dxfileconverter.py

@@ -143,7 +143,7 @@ class _DxFileToNxConverter(BaseConverter):
             output_file = os.path.join(
                 os.path.dirname(input_file),
                 os.path.basename(input_file_basename)
-                + self._configuration.file_extension,
+                + self._configuration.file_extension.value,
             )
         else:
             output_file = os.path.abspath(self.output_file)
@@ -329,7 +329,9 @@ class _DxFileToNxConverter(BaseConverter):
         self._copy_count_time()
 
         # start time
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             if "file_creation_datetime" in input_entry:
                 with HDF5File(self.output_file, mode="a") as h5f:
                     root_grp = h5f.require_group(self.output_entry)
@@ -365,7 +367,9 @@ class _DxFileToNxConverter(BaseConverter):
                         f"Failed to get 'count_time' / 'exposure_period'. reason is {e}"
                     )
 
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             if "measurement" in input_entry:
                 measurement_grp = input_entry["measurement"]
                 if "detector" in measurement_grp:
@@ -469,7 +473,9 @@ class _DxFileToNxConverter(BaseConverter):
         n_proj = 0
         # handle darks
         try:
-            with DatasetReader(self._data_darks_url) as dark_dataset:
+            with DatasetReader(
+                self._data_darks_url, open_mode_for_reading="safe"
+            ) as dark_dataset:
                 n_dark = dark_dataset.shape[0]
             # FIXME: avoid keeping some file open. not clear why this is needed
             dark_dataset = None
@@ -491,7 +497,9 @@ class _DxFileToNxConverter(BaseConverter):
 
         # handle flats
         try:
-            with DatasetReader(self._data_flats_url) as flat_dataset:
+            with DatasetReader(
+                self._data_flats_url, open_mode_for_reading="safe"
+            ) as flat_dataset:
                 n_flat = flat_dataset.shape[0]
             # FIXME: avoid keeping some file open. not clear why this is needed
             flat_dataset = None
@@ -513,7 +521,9 @@ class _DxFileToNxConverter(BaseConverter):
 
         # handle projections
         try:
-            with DatasetReader(self._data_proj_url) as proj_dataset:
+            with DatasetReader(
+                self._data_proj_url, open_mode_for_reading="safe"
+            ) as proj_dataset:
                 n_proj = proj_dataset.shape[0]
             # FIXME: avoid keeping some file open. not clear why this is needed
             proj_dataset = None
@@ -554,7 +564,9 @@ class _DxFileToNxConverter(BaseConverter):
             self._path_nxtomo_attrs(root_grp)
 
     def _read_sample_detector_distance(self):
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             path = "/".join(
                 ("measurement", "instrument", "sample", "setup", "detector_distance")
             )
@@ -568,7 +580,9 @@ class _DxFileToNxConverter(BaseConverter):
             return distance * unit
 
     def _read_energy(self):
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             path = "/".join(("measurement", "instrument", "source", "energy"))
             dataset = input_entry[path]
             energy = float(h5py_read_dataset(dataset))
@@ -580,7 +594,9 @@ class _DxFileToNxConverter(BaseConverter):
             return energy * unit
 
     def _read_x_pixel_size(self):
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             path = "/".join(
                 ("measurement", "instrument", "detector", "actual_pixel_size_x")
             )
@@ -594,7 +610,9 @@ class _DxFileToNxConverter(BaseConverter):
             return pixel_size * unit
 
     def _read_y_pixel_size(self):
-        with EntryReader(self.input_root_url) as input_entry:
+        with EntryReader(
+            self.input_root_url, open_mode_for_reading="safe"
+        ) as input_entry:
             path = "/".join(
                 ("measurement", "instrument", "detector", "actual_pixel_size_y")
             )

nxtomomill-2.1.0.dev3/nxtomomill/converter/edf/_Blosc2Jp2kWrapper.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+import numpy
+from h5py import Dataset
+
+try:
+    import blosc2
+    import blosc2_grok
+except ImportError:
+    _has_blosc2 = False
+else:
+    _has_blosc2 = True
+
+
+class Blosc2Jp2kWrapper:
+    """Wrapper to apply JP2K conversion to a dataset"""
+
+    def __init__(self, dataset: Dataset, compression_params: dict):
+        self.dataset: Dataset = dataset
+        self.compression_params: dict = self.get_default_compression_params()
+        self.compression_params.update(compression_params)
+
+        blosc2_grok.set_params_defaults(
+            # codec=blosc2.Codec.GROK,
+            cod_format=blosc2_grok.GrkFileFmt.GRK_FMT_JP2,
+            num_threads=self.compression_params["nthreads"],
+            quality_mode="rates",
+            quality_layers=numpy.array(
+                [self.compression_params["cratio"]],
+                dtype=numpy.float64,
+            ),
+            verbose=True,
+        )
+
+    def __setitem__(self, name, value):
+        z, _, _ = name
+        Z, Y, X = self.dataset.shape
+
+        # Verify data type compatibility
+        if value.dtype not in [numpy.float32, numpy.float64, numpy.uint16]:
+            raise ValueError(f"Unsupported dtype: {value.dtype}")
+
+        b2im = blosc2.asarray(
+            value.reshape(1, Y, X),
+            chunks=(1, Y, X),
+            blocks=(1, Y, X),
+            cparams=self.compression_params,
+        )
+        cframe = b2im.schunk.to_cframe()
+        self.dataset.id.write_direct_chunk((z, 0, 0), cframe, filter_mask=0)
+
+    @classmethod
+    def get_default_compression_params(cls) -> dict:
+        if _has_blosc2:
+            return {
+                "codec": blosc2.Codec.GROK,
+                "filters": [],
+                "splitmode": blosc2.SplitMode.NEVER_SPLIT,
+                "nthreads": 4,
+                "cratio": 10,
+                "verbose": True,
+            }
+        else:
+            return {}

nxtomomill-2.1.0.dev3/nxtomomill/converter/hdf5/acquisition/_bliss_activeconfig.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import h5py
+from silx.io.utils import h5py_read_dataset
+
+__all__ = [
+    "BlissActiveConfig",
+]
+
+
+class BlissActiveConfig:
+    """
+    hold the bliss active config. Can retrieve the 'tomo_detector' that contains information about the optic
+    """
+
+    def __init__(self) -> None:
+        self._tomo_detector: None | tuple[str, ...] = None
+
+    @property
+    def tomo_detector(self) -> tuple[str, ...] | None:
+        return self._tomo_detector
+
+    @staticmethod
+    def from_hdf5_group(group: h5py.Group):
+        """
+        Create an instance of 'BlissActiveConfig' from a group (containing the bliss active configuration).
+        """
+        if not isinstance(group, h5py.Group):
+            raise TypeError
+
+        bliss_config = BlissActiveConfig()
+        if "tomo_detector" in group:
+            bliss_config._tomo_detector = h5py_read_dataset(group["tomo_detector"])
+        return bliss_config

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev3}/nxtomomill/converter/hdf5/acquisition/baseacquisition.py

@@ -64,6 +64,7 @@ class BaseAcquisition:
     )
 
     _INSTRUMENT_NAME_PATH = (
+        "instrument/name",
         "technique/saving/beamline",
         "instrument/title",
     )
@@ -98,14 +99,10 @@ class BaseAcquisition:
 
     _SOURCE_NAME = ("instrument/machine/name",)
 
-    _BEAMLINE_NAME = (
-        "instrument/name",
-        "instrument/title",
-    )
-
     _SOURCE_TYPE = ("instrument/machine/type",)
 
     _FRAME_FLIP_PATHS = (
+        "instrument/{active_config_tomo_detector}/data_axes",  # note: data_axes are stored in the optic and not in the camera name.
         "technique/detector/{detector_name}/flipping",
         "technique/detector/flipping",
     )
@@ -134,6 +131,7 @@ class BaseAcquisition:
         The idea is to avoid asking him if """
         self._configuration = copy.deepcopy(configuration)
         """configuration to the conversion to be done. As we are modifying some values (keys...) and those depends on the acquisition context we need to copy it"""
+        self._configuration.output_entries = configuration.output_entries
         self._start_index = start_index
 
         # set of properties when an acquisition needs to copy dark / flat (used in the case of z-series version 3)
@@ -182,13 +180,27 @@ class BaseAcquisition:
                 output_file_basename.rstrip(possible_extension)
                 file_extension_ = possible_extension
 
-        def get_file_name_and_entry(index, divide_sub_files):
-            entry = "entry" + str(index).zfill(4)
+        def get_file_name_and_entry(
+            index: int, divide_sub_files: bool, entry: str | None
+        ) -> tuple[str, str]:
+            """
+            Docstring for get_file_name_and_entry
+
+            :param index: index of the entry in the input file
+            :param divide_sub_files: If True this mean the NXtomo will be split into several NXtomo. Case of multi-tomo.
+            :param entry: entry path if defined directly by the user.
+            """
+            if entry is None:
+                entry = "entry" + str(index).zfill(4)
+            elif not isinstance(entry, str):
+                raise TypeError(
+                    f"entry should be an instance of {str}. Got {type(entry)}"
+                )
 
             if self.configuration.single_file or not divide_sub_files:
                 en_output_file = self.configuration.output_file
             else:
-                ext = file_extension_ or self.configuration.file_extension
+                ext = file_extension_ or self.configuration.file_extension.value
                 file_name = (
                     os.path.splitext(output_file_basename)[0]
                     + "_"
@@ -212,8 +224,20 @@ class BaseAcquisition:
 
         result = []
         for i_nx_tomo, nx_tomo in enumerate(nx_tomos):
+            if self.configuration.output_entries is not None:
+                try:
+                    output_entry = self.configuration.output_entries.pop()
+                except IndexError:
+                    raise ValueError(
+                        "Not enough 'output_entries' provided for the number of nxtomo generated."
+                    )
+            else:
+                output_entry = None
+
             output_file, data_path = get_file_name_and_entry(
-                (shift_entry + i_nx_tomo), divide_sub_files=divide_into_sub_files
+                (shift_entry + i_nx_tomo),
+                divide_sub_files=divide_into_sub_files,
+                entry=output_entry,
             )
             output_file = abspath(os.path.relpath(output_file, os.getcwd()))
             output_file = abspath(output_file)
@@ -227,6 +251,11 @@ class BaseAcquisition:
                 _logger.warning(
                     f"No frame found for NXtomo number {i_nx_tomo}. Won't write any output for it"
                 )
+                if self.configuration.output_entries is not None:
+                    # append back output_entry. Will be the next one to (re)pop.
+                    # Because of sequence of sequence we might consume one entry name for nothing...
+                    self.configuration.output_entries.append(output_entry)
+
                 continue
             vs_0 = nx_tomo.instrument.detector.data[0]
             if nx_tomo.detector_data_is_defined_by_url():
@@ -252,7 +281,10 @@ class BaseAcquisition:
                         and self._copy_frames[url.path()]
                     ):
                         new_url = embed_url(url, output_file=output_file)
-                        n_vs, _, _ = from_data_url_to_virtual_source(new_url)
+                        n_vs, _, _ = from_data_url_to_virtual_source(
+                            new_url,
+                            open_mode_for_reading=self.configuration.open_mode_for_reading,
+                        )
                         new_vs.append(n_vs)
                     else:
                         new_vs.append(vs)
@@ -262,7 +294,11 @@ class BaseAcquisition:
             if input_file_path is not None:
                 nx_tomo.bliss_original_files = (abspath(input_file_path),)
             # save data
-            nx_tomo.save(file_path=output_file, data_path=data_path)
+            nx_tomo.save(
+                file_path=output_file,
+                data_path=data_path,
+                use_abs_path_for_vds=self.configuration.use_abs_path_for_vds,
+            )
             # check rotation angle
             if nx_tomo.sample.rotation_angle is not None:
                 unique_angles = numpy.unique(
@@ -299,7 +335,10 @@ class BaseAcquisition:
         raise NotImplementedError("Base class")
 
     def read_entry(self):
-        return EntryReader(self._root_url)
+        return EntryReader(
+            self._root_url,
+            open_mode_for_reading=self.configuration.open_mode_for_reading,
+        )
 
     def is_different_sequence(self, entry):
         """
@@ -314,7 +353,9 @@ class BaseAcquisition:
             return False
 
         def read_sample_node(url: DataUrl) -> str | None:
-            with EntryReader(url=url) as entry:
+            with EntryReader(
+                url=url, open_mode_for_reading=self.configuration.open_mode_for_reading
+            ) as entry:
                 sample_node = self._get_sample_node(entry)
                 if sample_node is None:
                     return None
@@ -504,7 +545,10 @@ class BaseAcquisition:
             _logger.warning("no root url. Unable to read rotation motor")
             return None
         else:
-            with EntryReader(self._root_url) as entry:
+            with EntryReader(
+                self._root_url,
+                open_mode_for_reading=self.configuration.open_mode_for_reading,
+            ) as entry:
                 for motor_path in self._TECHNIQUE_MOTOR_PATHS:
                     if motor_path in entry:
                         try:
@@ -777,7 +821,10 @@ class BaseAcquisition:
             _logger.warning("no root url. Unable to read detector roi")
             return None
         else:
-            with EntryReader(self._root_url) as entry:
+            with EntryReader(
+                self._root_url,
+                open_mode_for_reading=self.configuration.open_mode_for_reading,
+            ) as entry:
                 for roi_path in self._DETECTOR_ROI:
                     if roi_path in entry:
                         try: