tomwer 1.2.1__py3-none-any.whl → 1.3.12__py3-none-any.whl

tomwer/core/process/task.py

@@ -46,7 +46,9 @@ from ewokscore.task import Task as _EwoksTask
 from ewokscore.taskwithprogress import TaskWithProgress as _EwoksTaskWithProgress
 from silx.io.dictdump import dicttoh5, h5todict
 from silx.io.utils import h5py_read_dataset
+from silx.io.utils import open as open_hdf5
 from tomoscan.io import HDF5File
+from tomwer.core.utils.locker import FileLockerManager
 
 
 _logger = logging.getLogger(__name__)
@@ -173,19 +175,20 @@ class BaseProcessInfo:
         :type interpretations: Union[None, dict
         """
         assert process_file is not None
-        try:
-            self._register_process(
-                process_file=process_file,
-                entry=entry,
-                process=self,
-                configuration=configuration,
-                results=results,
-                process_index=process_index,
-                interpretations=interpretations,
-                overwrite=overwrite,
-            )
-        except IOError as e:
-            _logger.error(e)
+        with FileLockerManager().get_lock(file_name=process_file):
+            try:
+                self._register_process(
+                    process_file=process_file,
+                    entry=entry,
+                    process=self,
+                    configuration=configuration,
+                    results=results,
+                    process_index=process_index,
+                    interpretations=interpretations,
+                    overwrite=overwrite,
+                )
+            except IOError as e:
+                _logger.error(e)
 
     @staticmethod
     def _register_process(
@@ -373,33 +376,38 @@ class BaseProcessInfo:
         :return: list of _process_desc(sequence_index, configuration, results)
         :rtype: list
         """
-        processes = []
-        with HDF5File(process_file, mode="r", swmr=True) as h5f:
+        # retrieve process to load
+        with open_hdf5(process_file) as h5f:
             if entry is None:
                 if len(h5f.keys()) > 0:
                     root = h5f[list(h5f.keys())[0]]
                 else:
                     _logger.warning("no process find")
+                    return []
             else:
                 root = h5f[entry]
+
+            processes_to_load = {}
             for process in root.keys():
                 nx_process = root[process]
                 if (
                     h5py_read_dataset(nx_process["program"])
                     == process_type.program_name()
                 ):
-                    p_order = h5py_read_dataset(nx_process["sequence_index"])
-                    config = h5todict(
-                        process_file, "/".join((nx_process.name, "configuration"))
-                    )
-                    results = h5todict(
-                        process_file, "/".join((nx_process.name, "results"))
-                    )
-                    processes.append(
-                        _process_desc(
-                            process_order=p_order, configuration=config, results=results
-                        )
+                    processes_to_load[nx_process.name] = h5py_read_dataset(
+                        nx_process["sequence_index"]
                     )
+
+        # load process
+        processes = []
+        for process_name, p_order in processes_to_load.items():
+            config = h5todict(process_file, "/".join((process_name, "configuration")))
+            results = h5todict(process_file, "/".join((process_name, "results")))
+            processes.append(
+                _process_desc(
+                    process_order=p_order, configuration=config, results=results
+                )
+            )
         return processes
 
 

tomwer/core/process/test/test_axis.py

@@ -34,6 +34,7 @@ import tempfile
 import unittest
 
 import h5py
+from tomoscan.io import get_swmr_mode
 import numpy
 from silx.io.utils import h5py_read_dataset
 
@@ -42,8 +43,8 @@ from tomwer.core.process.reconstruction.axis.params import AxisRP
 from tomwer.core.process.task import Task
 from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
-from tomwer.core.utils.scanutils import MockEDF, MockHDF5
-from tomwer.tests.utils import UtilsTest
+from tomwer.core.utils.scanutils import MockEDF, MockNXtomo
+from tomwer.tests.datasets import TomwerCIDatasets
 
 from ..reconstruction.axis.axis import AxisTask
 
@@ -93,9 +94,9 @@ class TestAxisIO(unittest.TestCase):
                     )
 
                     # patch the axis process
-                    axis_process._CALCULATIONS_METHODS[
-                        AxisMode.centered
-                    ] = TestAxisIO._random_calc
+                    axis_process._CALCULATIONS_METHODS[AxisMode.centered] = (
+                        TestAxisIO._random_calc
+                    )
 
                     axis_process.run()
                     out = axis_process.outputs.data
@@ -131,7 +132,7 @@ class TestAxis(unittest.TestCase):
         axis_process.run()
         self.assertTrue(os.path.exists(scan.process_file))
 
-        with h5py.File(scan.process_file, "r", swmr=True) as h5f:
+        with h5py.File(scan.process_file, "r", swmr=get_swmr_mode()) as h5f:
             self.assertTrue("entry" in h5f)
             self.assertTrue("tomwer_process_0" in h5f["entry"])
             group_axis = h5f["entry"]["tomwer_process_0"]
@@ -163,7 +164,7 @@ class TestAxis(unittest.TestCase):
         """
         self.tempdir = tempfile.mkdtemp()
         dim = 10
-        mock = MockHDF5(
+        mock = MockNXtomo(
             scan_path=self.tempdir, n_proj=10, n_ini_proj=10, scan_range=180, dim=dim
         )
         mock.add_alignment_radio(index=10, angle=90)
@@ -183,7 +184,7 @@ class TestAxis(unittest.TestCase):
         # make sure center of position has been computed
         self.assertTrue(os.path.exists(scan.process_file))
 
-        with h5py.File(scan.process_file, "r", swmr=True) as h5f:
+        with h5py.File(scan.process_file, "r", swmr=get_swmr_mode()) as h5f:
             self.assertTrue("entry" in h5f)
             self.assertTrue("tomwer_process_0" in h5f["entry"])
             group_axis = h5f["entry"]["tomwer_process_0"]
@@ -234,8 +235,8 @@ class TestAxisRP(unittest.TestCase):
             self.axis_rp.set_position_frm_par_file(existing_empty_file, force=False)
         self.assertEqual(self.axis_rp.value_ref_tomwer, old_value)
 
-        valid_par_file = os.path.join(
-            UtilsTest.getEDFDataset("scan_3_"), "scan_3_slice.par"
+        valid_par_file = TomwerCIDatasets.get_dataset(
+            "edf_datasets/scan_3_/scan_3_slice.par"
         )
         assert os.path.isfile(valid_par_file)
         self.axis_rp.set_position_frm_par_file(valid_par_file, force=False)
@@ -255,12 +256,12 @@ class TestSinogramAlgorithm(unittest.TestCase):
     def setUp(self) -> None:
         self.tmp_folder = tempfile.mkdtemp()
         dim = 512
-        self.scan = MockHDF5(
+        self.scan = MockNXtomo(
             scan_path=self.tmp_folder,
             n_proj=10,
             n_ini_proj=10,
             create_ini_dark=True,
-            create_final_ref=True,
+            create_final_flat=True,
             scan_range=360,
             dim=dim,
         ).scan

tomwer/core/process/test/test_dark_and_flat.py

@@ -38,10 +38,10 @@ from tomoscan.esrf.scan.utils import get_data
 
 from tomwer.core.process.reconstruction.darkref.params import DKRFRP
 from tomwer.core.process.reconstruction.darkref.params import ReduceMethod as cMethod
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
-from tomwer.core.utils.scanutils import MockEDF, MockHDF5
-from tomwer.tests.utils import UtilsTest
+from tomwer.core.utils.scanutils import MockEDF, MockNXtomo
+from tomwer.tests.datasets import TomwerCIDatasets
 
 from ..reconstruction.darkref.darkrefs import DarkRefsTask
 from ..reconstruction.darkref.darkrefscopy import DarkRefsCopy
@@ -56,7 +56,9 @@ class TestDarkRefIO(unittest.TestCase):
         self.scan_edf = MockEDF.mockScan(
             scanID=self.scan_folder, nRadio=10, nRecons=1, nPagRecons=4, dim=10
         )
-        self.mock_hdf5 = MockHDF5(scan_path=self.scan_folder, n_proj=10, n_pag_recons=0)
+        self.mock_hdf5 = MockNXtomo(
+            scan_path=self.scan_folder, n_proj=10, n_pag_recons=0
+        )
         self.scan_hdf5 = self.mock_hdf5.scan
 
         self.recons_params = DKRFRP()
@@ -102,7 +104,7 @@ class TestDarkRefCopyIO(unittest.TestCase):
         self.scan_edf = MockEDF.mockScan(
             scanID=self.scan_folder, nRadio=10, nRecons=1, nPagRecons=4, dim=10
         )
-        self.scan_hdf5 = MockHDF5(
+        self.scan_hdf5 = MockNXtomo(
             scan_path=self.scan_folder, n_proj=10, n_pag_recons=0
         ).scan
         self.recons_params = DKRFRP()
@@ -256,9 +258,10 @@ class TestDarkRefNx(unittest.TestCase):
         self.scan_folder = tempfile.mkdtemp()
         self._file_path = os.path.join(self.scan_folder, dataset_name)
         shutil.copyfile(
-            src=UtilsTest.getH5Dataset(folderID=dataset_name), dst=self._file_path
+            src=TomwerCIDatasets.get_dataset(f"h5_datasets/{dataset_name}"),
+            dst=self._file_path,
         )
-        self.scan = HDF5TomoScan(scan=self._file_path, entry="entry0000")
+        self.scan = NXtomoScan(scan=self._file_path, entry="entry0000")
         self.assertFalse(os.path.exists(self.scan.process_file))
         self.recons_params = DKRFRP()
         self.recons_params.overwrite_dark = True

tomwer/core/process/test/test_data_transfer.py

@@ -40,17 +40,19 @@ import numpy
 import pytest
 from nxtomomill.converter import from_h5_to_nx
 from nxtomomill.io.config.hdf5config import TomoHDF5Config
-from tomoscan.esrf.scan.hdf5scan import ImageKey
+from nxtomo.nxobject.nxdetector import ImageKey
+
+from silx.io.utils import open as open_hdf5
 from tomoscan.io import HDF5File
 from tomoscan.validator import is_valid_for_reconstruction
 
 from tomwer.core.process.control.datalistener import DataListener
 from tomwer.core.process.control.scantransfer import ScanTransferTask
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.utils.scanutils import MockEDF
 from tomwer.synctools.rsyncmanager import RSyncManager
-from tomwer.tests.utils import UtilsTest
+from tomwer.tests.datasets import TomwerCIDatasets
 
 
 class TestDataTransferIO(unittest.TestCase):
@@ -119,7 +121,7 @@ class TestBlissDataTransfer(unittest.TestCase):
         self.input_dir = tempfile.mkdtemp()
         self.output_dir = tempfile.mkdtemp()
         shutil.copytree(
-            UtilsTest.getBlissDataset(folderID="sample"),
+            TomwerCIDatasets.get_dataset("bliss/sample"),
             os.path.join(self.input_dir, "sample"),
         )
 
@@ -204,7 +206,7 @@ class TestBlissDataTransfer(unittest.TestCase):
         configuration.file_extension = ".nx"
         from_h5_to_nx(configuration, input_callback=lambda *arg: 0)
         assert os.path.exists(output_file)
-        assert is_valid_for_reconstruction(HDF5TomoScan(output_file, entry="entry0000"))
+        assert is_valid_for_reconstruction(NXtomoScan(output_file, entry="entry0000"))
 
 
 @pytest.mark.skipif(not RSyncManager().has_rsync(), reason="requires rsync")
@@ -319,7 +321,7 @@ class NXTomoDataTransferBase(unittest.TestCase):
         return super().tearDown()
 
     def _test(self):
-        src_scan = HDF5TomoScan(self.nexus_file_path, entry="entry0000")
+        src_scan = NXtomoScan(self.nexus_file_path, entry="entry0000")
         assert is_valid_for_reconstruction(src_scan)
         process = ScanTransferTask(
             inputs={
@@ -334,9 +336,9 @@ class NXTomoDataTransferBase(unittest.TestCase):
         dst_scan = process.outputs.data
         assert dst_scan.master_file != self.nexus_file_path
         assert is_valid_for_reconstruction(dst_scan, check_values=True)
-        with HDF5File(src_scan.master_file, mode="r") as h5s_src:
+        with open_hdf5(src_scan.master_file) as h5s_src:
             src_dataset = h5s_src["entry0000/detector/data"][...]
-        with HDF5File(dst_scan.master_file, mode="r") as h5s_dst:
+        with open_hdf5(dst_scan.master_file) as h5s_dst:
             dst_dataset = h5s_dst["entry0000/detector/data"][...]
         assert numpy.array_equal(src_dataset, dst_dataset)
 

tomwer/core/process/test/test_nabu.py

@@ -41,7 +41,7 @@ from tomwer.core.process.reconstruction.nabu.nabuslices import NabuSliceMode
 from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.scanutils import MockEDF
-from tomwer.tests.utils import UtilsTest
+from tomwer.tests.datasets import TomwerCIDatasets
 
 
 class TestNabuDescUtils(unittest.TestCase):
@@ -51,7 +51,7 @@ class TestNabuDescUtils(unittest.TestCase):
         unittest.TestCase.setUp(self)
         self.top_dir = tempfile.mkdtemp()
         self.dataset = "scan_3_"
-        data_test_dir = UtilsTest.getEDFDataset(self.dataset)
+        data_test_dir = TomwerCIDatasets.get_dataset(f"edf_datasets/{self.dataset}")
         scan_folder = os.path.join(self.top_dir, self.dataset)
         shutil.copytree(data_test_dir, scan_folder)
         self.scan = ScanFactory.create_scan_object(scan_folder)
@@ -73,7 +73,7 @@ class TestNabuIni(unittest.TestCase):
     def setUp(self) -> None:
         self.dataset = "scan_3_"
         self._root_dir = tempfile.mkdtemp()
-        data_test_dir = UtilsTest.getEDFDataset(self.dataset)
+        data_test_dir = TomwerCIDatasets.get_dataset("edf_datasets/scan_3_")
         scan_folder = os.path.join(self._root_dir, self.dataset)
         shutil.copytree(data_test_dir, scan_folder)
         self.scan = EDFTomoScan(scan_folder)
@@ -127,7 +127,7 @@ class TestNabuIni(unittest.TestCase):
         # check volume and slice .cfg files for nabu reconstruction are here
         for file_name in (
             os.path.basename(self.scan.path) + ".cfg",
-            os.path.basename(self.scan.path) + "slice_000001" + ".cfg",
+            os.path.basename(self.scan.path) + "slice_000001_plane_XY" + ".cfg",
         ):
             with self.subTest(file_name=file_name):
                 self.assertTrue(
@@ -151,7 +151,9 @@ class TestNabuIni(unittest.TestCase):
         # check volume and slice .cfg files for nabu reconstruction are here
         for file_name in (
             os.path.basename(self.scan.path) + "pag_db0100.cfg",
-            os.path.basename(self.scan.path) + "slice_pag_000001_db0100" + ".cfg",
+            os.path.basename(self.scan.path)
+            + "slice_pag_000001_db0100_plane_XY"
+            + ".cfg",
         ):
             with self.subTest(file_name=file_name):
                 self.assertTrue(
@@ -179,10 +181,13 @@ class TestNabuIni(unittest.TestCase):
                 os.path.basename(self.scan.path)
                 + "slice_"
                 + str(slice_index).zfill(6)
+                + "_plane_XY"
                 + ".cfg"
             )
 
+        print("nabu_files are", nabu_files)
         for ini_file in ini_files:
+            print("check ini_file", ini_file)
             with self.subTest(file_name=ini_file):
                 self.assertTrue(
                     os.path.join(self._nabu_cfg_folder, ini_file) in nabu_files
@@ -220,10 +225,12 @@ class TestNabuIni(unittest.TestCase):
                 + ".cfg"
             )
             # add slice ini file
+            # note: plane is only specify for slice.
             ini_files.append(
                 os.path.basename(self.scan.path)
                 + "slice_pag_000001_db"
                 + str(int(pag_db)).zfill(4)
+                + "_plane_XY"
                 + ".cfg"
             )
 
@@ -268,6 +275,7 @@ class TestNabuIni(unittest.TestCase):
                     + str(slice_index).zfill(6)
                     + "_db"
                     + str(pag_db).zfill(4)
+                    + "_plane_XY"
                     + ".cfg"
                 )
 
@@ -356,7 +364,7 @@ class TestNabuAndAxis(unittest.TestCase):
     def setUp(self) -> None:
         self.dataset = "scan_3_"
         self._root_dir = tempfile.mkdtemp()
-        data_test_dir = UtilsTest.getEDFDataset(self.dataset)
+        data_test_dir = TomwerCIDatasets.get_dataset("edf_datasets/scan_3_")
         scan_folder = os.path.join(self._root_dir, self.dataset)
         shutil.copytree(data_test_dir, scan_folder)
         self.scan = EDFTomoScan(scan_folder)

tomwer/core/process/test/test_normalization.py

@@ -36,7 +36,7 @@ import numpy
 from tomoscan.normalization import Method
 
 from tomwer.core.process.reconstruction.normalization import normalization, params
-from tomwer.core.utils.scanutils import MockHDF5
+from tomwer.core.utils.scanutils import MockNXtomo
 
 
 class TestNormalization(unittest.TestCase):
@@ -47,15 +47,15 @@ class TestNormalization(unittest.TestCase):
     def setUp(self) -> None:
         self.tempdir = tempfile.mkdtemp()
         dim = 100
-        self.scan = MockHDF5(
+        self.scan = MockNXtomo(
             scan_path=self.tempdir,
             n_proj=2,
             n_ini_proj=2,
             scan_range=180,
             dim=dim,
             create_ini_dark=False,
-            create_ini_ref=False,
-            create_final_ref=False,
+            create_ini_flat=False,
+            create_final_flat=False,
         ).scan
 
     def tearDown(self) -> None:

tomwer/core/scan/blissscan.py

@@ -54,7 +54,7 @@ import logging
 from typing import Optional
 
 from silx.io.utils import h5py_read_dataset
-from tomoscan.io import HDF5File
+from silx.io.utils import open as open_hdf5
 
 _logger = logging.getLogger(__name__)
 
@@ -203,7 +203,7 @@ class BlissScan:
                     return True
             return False
 
-        with HDF5File(file_path, mode="r", swmr=True) as h5s:
+        with open_hdf5(file_path) as h5s:
             if not isinstance(h5s, h5py.Group) or not isinstance(
                 h5s[entry], h5py.Group
             ):
@@ -227,7 +227,7 @@ class BlissScan:
             return tuple()
         else:
             res = []
-            with HDF5File(file_path, mode="r", swmr=True) as h5s:
+            with open_hdf5(file_path) as h5s:
                 for entry in h5s:
                     if BlissScan.is_bliss_valid_entry(file_path=file_path, entry=entry):
                         res.append(entry)

tomwer/core/scan/edfscan.py

@@ -48,7 +48,7 @@ from tomoscan.esrf.identifier.edfidentifier import (
 )
 from tomoscan.esrf.scan.edfscan import EDFTomoScan as _tsEDFTomoScan
 from tomoscan.esrf.scan.utils import get_data
-from tomoscan.scanbase import FOV
+from nxtomo.nxobject.nxdetector import FOV
 
 from tomwer.core.process.reconstruction.darkref.settings import (
     DARKHST_PREFIX,
@@ -85,6 +85,9 @@ class EDFTomoScanIdentifier(_EDFTomoScanIdentifier, DatasetIdentifier):
         """used for processview header tooltip for now"""
         return self.to_str()
 
+    def short_description(self) -> str:
+        return f"scan: {os.path.basename(os.path.basename(self.folder))} (EDF)"
+
 
 class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
     """
@@ -278,9 +281,9 @@ class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
                                     files.append(gfile)
                     if f.endswith(".vol"):
                         if withIndex is True:
-                            files[
-                                EDFTomoScan.get_index_reconstructed(f, scanID)
-                            ] = os.path.join(scanID, f)
+                            files[EDFTomoScan.get_index_reconstructed(f, scanID)] = (
+                                os.path.join(scanID, f)
+                            )
                         else:
                             files.append(os.path.join(scanID, f))
             return files
@@ -549,12 +552,12 @@ class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
             file_prefix=self.dataset_basename,
             metadata={
                 "name": self.path,
-                "creation_time": datetime.fromtimestamp(stat.st_ctime)
-                if stat
-                else None,
-                "modification_time": datetime.fromtimestamp(stat.st_ctime)
-                if stat
-                else None,
+                "creation_time": (
+                    datetime.fromtimestamp(stat.st_ctime) if stat else None
+                ),
+                "modification_time": (
+                    datetime.fromtimestamp(stat.st_ctime) if stat else None
+                ),
             },
         )