nxtomomill 2.1.0.dev1__tar.gz → 2.1.0.dev2__tar.gz

{nxtomomill-2.1.0.dev1/nxtomomill.egg-info → nxtomomill-2.1.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nxtomomill
-Version: 2.1.0.dev1
+Version: 2.1.0.dev2
 Summary: applications and library to convert raw format to NXtomo format
 Author-email: Henri Payno <henri.payno@esrf.fr>, Pierre Paleo <pierre.paleo@esrf.fr>, Pierre-Olivier Autran <pierre-olivier.autran@esrf.fr>, Jérôme Lesaint <jerome.lesaint@esrf.fr>, Alessandro Mirone <mirone@esrf.fr>
 License: 

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/app/h52nx.py

@@ -207,8 +207,8 @@ def main(argv):
     )
     parser.add_argument(
         "--read-mode",
-        default="locking",
-        help="Options when reading bliss file. Valid values are 'locking', 'swmr' and 'locking_free'. 'locking' is the default h5py read mode.",
+        default="robust",
+        help="Options when reading bliss file. Valid values are 'locking', 'swmr' and 'safe'. 'robust' is the default h5py read mode.",
     )
     # scan titles
     parser.add_argument(
@@ -284,6 +284,12 @@ def main(argv):
         f"are: {_getPossibleInputParams()}. Should be added at the end of the command line because "
         "will try to cover all text set after this option.",
     )
+    # output entries
+    parser.add_argument(
+        "--output-entries",
+        default=None,
+        help="Provide names of the output entries (HDF5 group path).",
+    )
 
     # config file
     parser.add_argument(
@@ -303,10 +309,10 @@ def main(argv):
         callback_det_sel = None
 
     # define reading mode
-    read_mode = os.getenv("NXTOMOMILL_HDF5_READ_MODE", "locking")
-    if options.read_mode not in ("locking", "locking_free", "swmr"):
+    read_mode = os.getenv("NXTOMOMILL_HDF5_READ_MODE", "robust")
+    if options.read_mode not in ("robust", "safe", "swmr"):
         raise ValueError(
-            f"Invalid read mode: {read_mode}. Valid values are 'locking', 'locking_free' and 'swmr'"
+            f"Invalid read mode: {read_mode}. Valid values are 'robust', 'safe' and 'swmr'"
         )
     os.environ["NXTOMOMILL_HDF5_READ_MODE"] = options.read_mode
 

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/converter/hdf5/acquisition/baseacquisition.py

@@ -134,6 +134,7 @@ class BaseAcquisition:
         The idea is to avoid asking him if """
         self._configuration = copy.deepcopy(configuration)
         """configuration to the conversion to be done. As we are modifying some values (keys...) and those depends on the acquisition context we need to copy it"""
+        self._configuration.output_entries = configuration.output_entries
         self._start_index = start_index
 
         # set of properties when an acquisition needs to copy dark / flat (used in the case of z-series version 3)
@@ -182,8 +183,22 @@ class BaseAcquisition:
                 output_file_basename.rstrip(possible_extension)
                 file_extension_ = possible_extension
 
-        def get_file_name_and_entry(index, divide_sub_files):
-            entry = "entry" + str(index).zfill(4)
+        def get_file_name_and_entry(
+            index: int, divide_sub_files: bool, entry: str | None
+        ) -> tuple[str, str]:
+            """
+            Docstring for get_file_name_and_entry
+
+            :param index: index of the entry in the input file
+            :param divide_sub_files: If True this mean the NXtomo will be split into several NXtomo. Case of multi-tomo.
+            :param entry: entry path if defined directly by the user.
+            """
+            if entry is None:
+                entry = "entry" + str(index).zfill(4)
+            elif not isinstance(entry, str):
+                raise TypeError(
+                    f"entry should be an instance of {str}. Got {type(entry)}"
+                )
 
             if self.configuration.single_file or not divide_sub_files:
                 en_output_file = self.configuration.output_file
@@ -212,8 +227,20 @@ class BaseAcquisition:
 
         result = []
         for i_nx_tomo, nx_tomo in enumerate(nx_tomos):
+            if self.configuration.output_entries is not None:
+                try:
+                    output_entry = self.configuration.output_entries.pop()
+                except IndexError:
+                    raise ValueError(
+                        "Not enough 'output_entries' provided for the number of nxtomo generated."
+                    )
+            else:
+                output_entry = None
+
             output_file, data_path = get_file_name_and_entry(
-                (shift_entry + i_nx_tomo), divide_sub_files=divide_into_sub_files
+                (shift_entry + i_nx_tomo),
+                divide_sub_files=divide_into_sub_files,
+                entry=output_entry,
             )
             output_file = abspath(os.path.relpath(output_file, os.getcwd()))
             output_file = abspath(output_file)
@@ -227,6 +254,11 @@ class BaseAcquisition:
                 _logger.warning(
                     f"No frame found for NXtomo number {i_nx_tomo}. Won't write any output for it"
                 )
+                if self.configuration.output_entries is not None:
+                    # append back output_entry. Will be the next one to (re)pop.
+                    # Because of sequence of sequence we might consume one entry name for nothing...
+                    self.configuration.output_entries.append(output_entry)
+
                 continue
             vs_0 = nx_tomo.instrument.detector.data[0]
             if nx_tomo.detector_data_is_defined_by_url():

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/converter/hdf5/acquisition/tests/test_acquisition_utils.py

@@ -30,6 +30,7 @@ from nxtomomill.converter.hdf5.acquisition.tests.bliss_tomo_datasets import (  #
     holotomo2_simu_dataset,
 )
 from nxtomomill.converter.hdf5.acquisition.utils._scan_type_finder import ScanTypeFinder
+from nxtomomill.models.h52nx.h52nx import H52nxModel
 
 
 def test_deduce_machine_current():
@@ -210,7 +211,7 @@ def test_group_z_series(tmp_path):
                             data_path=f"{i_scan}.1",
                             scheme="silx",
                         ),
-                        configuration={},
+                        configuration=H52nxModel(),
                         detector_sel_callback=None,
                         start_index=i_scan,
                     )

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/converter/hdf5/acquisition/tests/test_holotomo.py

@@ -54,12 +54,19 @@ def test_holotomo_conversion_id16a(tmp_path):
     configuration.no_master_file = False
     configuration.translation_y_keys = _cast_keys_section_to_tuple(("sy+spy",))
     configuration.translation_z_keys = _cast_keys_section_to_tuple(("sz+spz",))
+    configuration.output_entries = (
+        "my_entry_0000",
+        "my_entry_0001",
+    )
 
     result = converter.from_h5_to_nx(configuration=configuration)
+
     assert os.path.exists(output_file)
 
     # check we have our 2 NXtomo are converted with (two distances).
     assert len(result) == 2
+    assert result[0][1] == "my_entry_0000"
+    assert result[1][1] == "my_entry_0001"
     # expected:
     nx_tomos = [NXtomo().load(file_path, entry) for (file_path, entry) in result]
     n_proj_per_nx_tomo = 1000

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/converter/hdf5/hdf5converter.py

@@ -19,7 +19,6 @@ from tomoscan.utils.io import filter_esrf_mounting_points
 
 from nxtomomill.converter.hdf5.acquisition.utils import group_series
 from nxtomomill.converter.baseconverter import BaseConverter
-from nxtomomill.converter.hdf5.acquisition.baseacquisition import _ask_for_file_removal
 from nxtomomill.converter.hdf5.acquisition.multitomo import MultiTomoAcquisition
 from nxtomomill.models.h52nx._acquisitionstep import AcquisitionStep
 from nxtomomill.models.h52nx import H52nxModel
@@ -329,6 +328,13 @@ class _H5ToNxConverter(BaseConverter):
             _logger.info(f"create link in {self.configuration.output_file}")
             for en_output_file, entry in res:
                 with HDF5File(self.configuration.output_file, "a") as master_file:
+                    if entry in master_file:
+                        if self.configuration.overwrite:
+                            del master_file[entry]
+                        else:
+                            raise OSError(
+                                f"Data path {entry} already exists. Define overwrite option to overwrite existing file and entries."
+                            )
                     link_file = os.path.relpath(
                         filter_esrf_mounting_points(en_output_file),
                         filter_esrf_mounting_points(
@@ -358,15 +364,6 @@ class _H5ToNxConverter(BaseConverter):
         dir_name = os.path.dirname(abspath(output_file))
         if not os.path.exists(dir_name):
             os.makedirs(os.path.dirname(abspath(output_file)))
-        elif os.path.exists(output_file):
-            if self.configuration.overwrite is True:
-                _logger.warning(f"{output_file} will be removed")
-                _logger.info(f"remove {output_file}")
-                os.remove(output_file)
-            elif not _ask_for_file_removal(output_file):
-                raise OSError(f"unable to overwrite {output_file}, exit")
-            else:
-                os.remove(output_file)
         if not os.access(dir_name, os.W_OK):
             raise OSError(f"You don't have rights to write on {dir_name}")
 

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/io/config/confighandler.py

@@ -149,6 +149,7 @@ class TomoHDF5ConfigHandler:
             "flat_titles",
             "projection_titles",
             "alignment_titles",
+            "output_entries",
         ):
             opt_value = getattr(options, opt_name)
             if opt_value is not None:

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/io/config/tests/test_confighandler.py

@@ -42,6 +42,7 @@ class _ARParseMock(object):
         self.file_extension = None
         self.field_of_view = None
         self.sample_detector_distance_keys = None
+        self.output_entries = None
 
 
 def test_creation_from_config_file(tmp_path):

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/models/h52nx/general_section.py

@@ -1,10 +1,15 @@
 from __future__ import annotations
 
+from collections import deque
 from pydantic import Field, field_serializer, field_validator, ConfigDict
 
 from nxtomo.nxobject.nxdetector import FieldOfView
 
-from nxtomomill.models.utils import filter_str_def, convert_str_to_bool
+from nxtomomill.models.utils import (
+    filter_str_def,
+    convert_str_to_bool,
+    convert_str_to_tuple,
+)
 from ..base.general_section import GeneralSection as _GeneralSectionBase
 
 
@@ -12,6 +17,9 @@ class GeneralSection(_GeneralSectionBase):
     model_config = ConfigDict(validate_assignment=True, validate_by_name=True)
 
     input_file: str | None = Field(default=None, description="Input file path.")
+    output_entries: deque[str] | None = Field(
+        default=None, description="Output of the conversion"
+    )
     raises_error: bool = Field(
         default=False, description="Raise an error when encountered."
     )
@@ -130,3 +138,28 @@ class GeneralSection(_GeneralSectionBase):
     @classmethod
     def cast_general_section_to_bool(cls, value: str | bool | None) -> bool:
         return convert_str_to_bool(value)
+
+    @field_validator(
+        "output_entries",
+        mode="plain",
+    )
+    @classmethod
+    def cast_output_entries(
+        cls, value: str | tuple[str] | deque[str]
+    ) -> deque[str] | None:
+        if value is None:
+            return None
+        if isinstance(value, str):
+            value = filter_str_def(value)
+        elif isinstance(value, deque):
+            return value
+        values = convert_str_to_tuple(value)
+        if values is None or len(values) == 0:
+            return None
+        if len(values) != len(set(values)):
+            raise ValueError("several values are identical.")
+        res = deque()
+        # move it to a 'deque' to simplify usage
+        # last element added will be the first one pop. This is why we need to invert it.
+        [res.append(v) for v in values[::-1]]
+        return res

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/models/h52nx/h52nx.py

@@ -54,6 +54,7 @@ class H52nxModel(
                 check_tomo_n=self.check_tomo_n,
                 mechanical_lr_flip=self.mechanical_lr_flip,
                 mechanical_ud_flip=self.mechanical_ud_flip,
+                output_entries=self.output_entries,
             ),
             keys_section=KeysSection(
                 valid_camera_names=self.valid_camera_names,

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/utils/hdf5.py

@@ -8,6 +8,7 @@ import pint
 import logging
 from .pintutils import get_unit
 from silx.io.utils import open as open_hdf5
+from silx.io.h5py_utils import File
 
 try:
     import hdf5plugin  # noqa F401
@@ -97,13 +98,13 @@ def get_dataset_unit(
     return get_unit(unit=unit, default=default, from_dataset=from_dataset)
 
 
-def _locking_free_read(filename: str):
+def _robust_read(filename: str):
     if not h5py.is_hdf5(filename):
         raise OSError(f"{filename} doesn't is not a valid hdf5 file")
     try:
-        return h5py.File(filename, "r", locking=False)
+        return File(filename, "r", locking=False)
     except OSError:
-        return h5py.File(filename, "r", libver="latest", swmr=True)
+        return File(filename, "r", libver="latest", swmr=True)
 
 
 def _swmr_read(filename: str):
@@ -113,14 +114,14 @@ def _swmr_read(filename: str):
 
 
 def read_hdf5(filename: str):
-    read_mode = os.getenv("NXTOMOMILL_HDF5_READ_MODE", "locking")
-    if read_mode not in ("locking", "locking_free", "swmr"):
+    read_mode = os.getenv("NXTOMOMILL_HDF5_READ_MODE", "robust")
+    if read_mode not in ("robust", "safe", "swmr"):
         raise ValueError(
-            f"Invalid read mode: {read_mode}. Valid values are 'locking', 'locking_free' and 'swmr'"
+            f"Invalid read mode: {read_mode}. Valid values are 'locking', 'safe' and 'swmr'"
         )
-    if read_mode == "locking":
+    if read_mode == "safe":
         return open_hdf5(filename)
-    elif read_mode == "locking_free":
-        return _locking_free_read(filename)
+    elif read_mode == "robust":
+        return _robust_read(filename)
     elif read_mode == "swmr":
         return _swmr_read(filename)

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill/version.py

@@ -57,7 +57,7 @@ MAJOR = 2
 MINOR = 1
 MICRO = 0
 RELEV = "dev"  # <16
-SERIAL = 1  # <16
+SERIAL = 2  # <16
 
 date = __date__
 

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2/nxtomomill.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nxtomomill
-Version: 2.1.0.dev1
+Version: 2.1.0.dev2
 Summary: applications and library to convert raw format to NXtomo format
 Author-email: Henri Payno <henri.payno@esrf.fr>, Pierre Paleo <pierre.paleo@esrf.fr>, Pierre-Olivier Autran <pierre-olivier.autran@esrf.fr>, Jérôme Lesaint <jerome.lesaint@esrf.fr>, Alessandro Mirone <mirone@esrf.fr>
 License: 

{nxtomomill-2.1.0.dev1 → nxtomomill-2.1.0.dev2}/nxtomomill.egg-info/top_level.txt

@@ -1,4 +1,3 @@
 dist
 doc
-html
 nxtomomill