tomwer 1.2.8__py3-none-any.whl → 1.3.0a0__py3-none-any.whl

tomwer/core/scan/scanfactory.py

@@ -36,17 +36,17 @@ from urllib.parse import urlparse
 
 import h5py
 from tomoscan.esrf.identifier.url_utils import split_path
-from tomoscan.esrf.scan.hdf5scan import HDF5TomoScan as _oHDF5TomoScan
 from tomoscan.factory import Factory as _oScanFactory
 from tomoscan.identifier import BaseIdentifier, ScanIdentifier, VolumeIdentifier
 from tomoscan.scanbase import TomoScanBase
 from tomoscan.tomoobject import TomoObject
+from nxtomo.application.nxtomo import NXtomo
 
 from tomwer.utils import docstring
 
 from .blissscan import BlissScan
 from .edfscan import EDFTomoScan, EDFTomoScanIdentifier
-from .hdf5scan import HDF5TomoScan, HDF5TomoScanIdentifier
+from .hdf5scan import NXtomoScan, NXtomoScanIdentifier
 
 _logger = logging.getLogger(__name__)
 
@@ -93,7 +93,7 @@ class ScanFactory(object):
                 if scheme == "edf":
                     identifier = EDFTomoScanIdentifier.from_str(identifier=identifier)
                 elif scheme == "hdf5":
-                    identifier = HDF5TomoScanIdentifier.from_str(identifier=identifier)
+                    identifier = NXtomoScanIdentifier.from_str(identifier=identifier)
                 else:
                     raise ValueError(f"Scheme {scheme} not recognized")
             else:
@@ -111,7 +111,7 @@ class ScanFactory(object):
                 raise NotImplementedError()
         elif scheme == "hdf5":
             if tomo_type == ScanIdentifier.TOMO_TYPE:
-                return HDF5TomoScan.from_identifier(identifier=identifier)
+                return NXtomoScan.from_identifier(identifier=identifier)
             else:
                 raise NotImplementedError()
         else:
@@ -135,7 +135,7 @@ class ScanFactory(object):
         if entry is not None and not entry.startswith("/"):
             entry = "/" + entry
         if os.path.isfile(scan_path) and ScanFactory.is_hdf5_tomo(scan_path):
-            valid_entries = _oHDF5TomoScan.get_valid_entries(scan_path)
+            valid_entries = NXtomo.get_valid_entries(scan_path)
             if entry is None:
                 if len(valid_entries) > 1:
                     _logger.warning(
@@ -148,7 +148,7 @@ class ScanFactory(object):
                     f"entry {entry} is invalid. Does it exists ? Is the "
                     f"file NXTomo compliant ?. Valid entry are {valid_entries}"
                 )
-            return HDF5TomoScan(scan=scan_path, entry=entry)
+            return NXtomoScan(scan=scan_path, entry=entry)
         elif ScanFactory.is_edf_tomo(scan_path):
             return EDFTomoScan(scan=scan_path)
         elif accept_bliss_scan and BlissScan.is_bliss_file(scan_path):
@@ -169,12 +169,12 @@ class ScanFactory(object):
         scan_path = scan_path.rstrip(os.path.sep)
         if EDFTomoScan.is_tomoscan_dir(scan_path):
             return (EDFTomoScan(scan=scan_path),)
-        elif HDF5TomoScan.is_tomoscan_dir(scan_path):
+        elif NXtomoScan.is_tomoscan_dir(scan_path):
             scans = []
-            master_file = HDF5TomoScan.get_master_file(scan_path=scan_path)
-            entries = HDF5TomoScan.get_valid_entries(master_file)
+            master_file = NXtomoScan.get_master_file(scan_path=scan_path)
+            entries = NXtomo.get_valid_entries(master_file)
             for entry in entries:
-                scans.append(HDF5TomoScan(scan=scan_path, entry=entry, index=None))
+                scans.append(NXtomoScan(scan=scan_path, entry=entry, index=None))
             return tuple(scans)
         elif accept_bliss_scan and BlissScan.is_bliss_file(scan_path):
             scans = []
@@ -201,8 +201,8 @@ class ScanFactory(object):
             )
         elif _dict[TomoScanBase.DICT_TYPE_KEY] == EDFTomoScan._TYPE:
             return EDFTomoScan(scan=None).load_from_dict(_dict)
-        elif _dict[TomoScanBase.DICT_TYPE_KEY] == HDF5TomoScan._TYPE:
-            return HDF5TomoScan.from_dict(_dict)
+        elif _dict[TomoScanBase.DICT_TYPE_KEY] == NXtomoScan._TYPE:
+            return NXtomoScan.from_dict(_dict)
         else:
             raise ValueError(
                 f"Scan type {_dict[TomoScanBase.DICT_TYPE_KEY]} is not managed"
@@ -253,4 +253,4 @@ class ScanFactory(object):
         if os.path.isfile(scan_path):
             return h5py.is_hdf5(scan_path)
         else:
-            return HDF5TomoScan.directory_contains_scan(scan_path)
+            return NXtomoScan.directory_contains_scan(scan_path)

tomwer/core/scan/test/test_future_scan.py

@@ -33,12 +33,12 @@ import asyncio
 import os
 
 from tomwer.core.futureobject import FutureTomwerObject
-from tomwer.core.utils.scanutils import MockHDF5
+from tomwer.core.utils.scanutils import MockNXtomo
 
 
 def test_simple_future_tomo_obj(tmpdir):
     """Simple test of the FutureTomwerScan API"""
-    scan = MockHDF5(
+    scan = MockNXtomo(
         scan_path=os.path.join(tmpdir, "scan_test"),
         n_proj=10,
         n_ini_proj=10,

tomwer/core/scan/test/test_h5.py

@@ -33,9 +33,9 @@ import shutil
 import tempfile
 import unittest
 
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
-from tomwer.core.utils.scanutils import MockHDF5
-from tomwer.tests.utils import UtilsTest
+from tomwer.core.scan.nxtomoscan import NXtomoScan
+from tomwer.core.utils.scanutils import MockNXtomo
+from tomwer.tests.datasets import TomwerCIDatasets
 
 
 class TestHDF5Scan(unittest.TestCase):
@@ -46,11 +46,13 @@ class TestHDF5Scan(unittest.TestCase):
         self._tmp_dir = tempfile.mkdtemp()
         self.dataset_file = os.path.join(self._tmp_dir, "frm_edftomomill_twoentries.nx")
         shutil.copyfile(
-            UtilsTest.getH5Dataset(folderID="frm_edftomomill_twoentries.nx"),
+            TomwerCIDatasets.get_dataset(
+                "h5_datasets/frm_edftomomill_twoentries.nx",
+            ),
             self.dataset_file,
         )
         assert os.path.isfile(self.dataset_file)
-        self.scan = HDF5TomoScan(scan=self.dataset_file, entry="entry0000")
+        self.scan = NXtomoScan(scan=self.dataset_file, entry="entry0000")
 
     def test_scan_dir(self):
         assert self.scan.scan_dir_name() == self._tmp_dir.split(os.sep)[-1]
@@ -58,7 +60,7 @@ class TestHDF5Scan(unittest.TestCase):
     def testFFInterval(self):
         """test the call to ff_interval"""
         scan_path = os.path.join(self._tmp_dir, "my_scan_1")
-        scan_1 = MockHDF5(
+        scan_1 = MockNXtomo(
             scan_path=scan_path,
             n_ini_proj=20,
             n_proj=20,
@@ -71,7 +73,7 @@ class TestHDF5Scan(unittest.TestCase):
         self.assertEqual(scan_1.ff_interval, 20)
 
         scan_path2 = os.path.join(self._tmp_dir, "my_scan_2")
-        scan_2 = MockHDF5(
+        scan_2 = MockNXtomo(
             scan_path=scan_path2,
             n_ini_proj=10,
             n_proj=10,
@@ -86,9 +88,9 @@ class TestHDF5Scan(unittest.TestCase):
 
 def test_working_directory():
     """test behavior of the working directory function"""
-    scan = HDF5TomoScan(scan=None, entry="my_entry")
+    scan = NXtomoScan(scan=None, entry="my_entry")
     assert scan.working_directory is None
-    scan = HDF5TomoScan(scan="/full/path/my_file.sh", entry="my_entry")
+    scan = NXtomoScan(scan="/full/path/my_file.sh", entry="my_entry")
     assert scan.working_directory == os.path.realpath("/full/path")
-    scan = HDF5TomoScan(scan="my_file.sh", entry="my_entry")
+    scan = NXtomoScan(scan="my_file.sh", entry="my_entry")
     assert scan.working_directory == os.path.realpath(".")

tomwer/core/scan/test/test_process_registration.py

@@ -35,7 +35,7 @@ import unittest
 from tomoscan.io import HDF5File
 
 from tomwer.core.process.task import Task
-from tomwer.core.utils.scanutils import MockHDF5
+from tomwer.core.utils.scanutils import MockNXtomo
 
 
 class TestProcessRegistration(unittest.TestCase):
@@ -62,7 +62,7 @@ class TestProcessRegistration(unittest.TestCase):
 
     def setUp(self):
         self.tmp_dir = tempfile.mkdtemp()
-        self.scan = MockHDF5(scan_path=self.tmp_dir, n_proj=2).scan
+        self.scan = MockNXtomo(scan_path=self.tmp_dir, n_proj=2).scan
 
     def tearDown(self):
         shutil.rmtree(self.tmp_dir)

tomwer/core/scan/test/test_scan.py

@@ -43,7 +43,7 @@ from tomwer.core.process.task import Task
 from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.scanutils import MockEDF
-from tomwer.tests.utils import UtilsTest
+from tomwer.tests.datasets import TomwerCIDatasets
 
 logging.disable(logging.INFO)
 
@@ -59,7 +59,9 @@ class TestScanFactory(unittest.TestCase):
             ScanFactory.create_scan_object(scan_dir)
 
     def test_scan_edf(self):
-        scan_dir = UtilsTest.getEDFDataset("test10")
+        scan_dir = TomwerCIDatasets.get_dataset(
+            "edf_datasets/test10",
+        )
         scan = ScanFactory.create_scan_object(scan_dir)
         self.assertTrue(isinstance(scan, EDFTomoScan))
 
@@ -102,7 +104,6 @@ class TestScanValidatorFindNabuFiles(unittest.TestCase):
                             ctf=ctf if delta_beta is not None else False,
                             db=delta_beta,
                         )
-                        print("file_name is", file_name)
                         with self.subTest(
                             slice_index=slice_index,
                             pag_db=delta_beta,

tomwer/core/settings.py

@@ -92,6 +92,7 @@ class SlurmSettingsMode(_Enum):
     CAST_VOLUME = "cast_volume"
     SLICE_RECONSTRUCTION = "slice_reconstruction"
     VOLUME_RECONSTRUCTION = "volume_reconstruction"
+    STITCHING = "stitching"
 
     @staticmethod
     def get_settings_class(mode):
@@ -106,6 +107,8 @@ class SlurmSettingsMode(_Enum):
             return DefaultSlurmSettingsSliceReconstruction
         elif mode is SlurmSettingsMode.VOLUME_RECONSTRUCTION:
             return DefaultSlurmSettingsVolumeReconstruction
+        elif mode is SlurmSettingsMode.STITCHING:
+            return DefaultSlurmSettingsStitching
         else:
             raise ValueError(f"{mode} not handled")
 
@@ -141,6 +144,12 @@ class SlurmSettings:
     """Slurm cluster project name. `scan`, `process` and `info` will be format.
     """
 
+    MODULES_TO_LOAD = tuple()  # in the future might be 'tomotools'. In this case we must remove the PYTHON_VENV
+
+    SBATCH_EXTRA_PARAMS = {
+        "export": "NONE",  # value to provide to sbatch --export={}
+    }
+
 
 class DefaultSlurmSettingsCastVolume(SlurmSettings):
     """
@@ -171,3 +180,14 @@ class DefaultSlurmSettingsVolumeReconstruction(SlurmSettings):
     MEMORY_PER_WORKER = 256  # memory in GB
 
     PARTITION = None
+
+
+class DefaultSlurmSettingsStitching(SlurmSettings):
+    """*
+    default proposed configuration for stitching
+    """
+    N_JOBS = 15
+
+    PARTITION = "nice-long"
+
+    N_GPUS_PER_WORKER = 0

tomwer/core/test/test_scanutils.py

@@ -31,22 +31,23 @@ import os
 import tempfile
 import unittest
 
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
-from tomwer.core.utils.scanutils import MockHDF5
+from nxtomo.application.nxtomo import NXtomo
+from tomwer.core.scan.nxtomoscan import NXtomoScan
+from tomwer.core.utils.scanutils import MockNXtomo
 
 
-class TestMockHDF5(unittest.TestCase):
-    """Test the MockHDF5 file"""
+class TestMockNXtomo(unittest.TestCase):
+    """Test the MockNXtomo file"""
 
     def test_creation(self):
         folder = tempfile.mkdtemp()
-        mock = MockHDF5(scan_path=folder, n_proj=10, n_ini_proj=10)
+        mock = MockNXtomo(scan_path=folder, n_proj=10, n_ini_proj=10)
         self.assertEqual(
             mock.scan_master_file,
             os.path.join(folder, os.path.basename(folder) + ".h5"),
         )
-        tomoScan = HDF5TomoScan(mock.scan_path, entry=mock.scan_entry)
-        self.assertEqual(len(HDF5TomoScan.get_valid_entries(mock.scan_master_file)), 1)
+        tomoScan = NXtomoScan(mock.scan_path, entry=mock.scan_entry)
+        self.assertEqual(len(NXtomo.get_valid_entries(mock.scan_master_file)), 1)
         tomoScan.update()
         self.assertEqual(tomoScan.scan_range, 360)
         self.assertEqual(len(tomoScan.projections), 10)

tomwer/core/test/test_utils.py

@@ -35,22 +35,29 @@ import unittest
 
 import numpy
 
-from tomwer.core import utils
+from tomwer.core.utils import spec as spec_utils
 from tomwer.core.utils.normalization import flatFieldCorrection
-from tomwer.core.utils.scanutils import MockEDF, MockHDF5
-from tomwer.tests.utils import UtilsTest
+from tomwer.core.utils.scanutils import MockEDF, MockNXtomo
+from tomwer.tests.datasets import TomwerCIDatasets
 
 
 class TestGetClosestEnergy(unittest.TestCase):
     def setUp(self):
         self.topSrcFolder = tempfile.mkdtemp()
         self.dataSetID = "scan_3_"
-        self.dataDir = UtilsTest.getEDFDataset(self.dataSetID)
+        self.dataDir = TomwerCIDatasets.get_dataset(
+            f"edf_datasets/{self.dataSetID}",
+        )
         self.sourceS3 = os.path.join(self.topSrcFolder, self.dataSetID)
         shutil.copytree(src=os.path.join(self.dataDir), dst=self.sourceS3)
 
         self.sourceT01 = os.path.join(self.topSrcFolder, "test01")
-        shutil.copytree(src=UtilsTest.getEDFDataset("test01"), dst=self.sourceT01)
+        shutil.copytree(
+            src=TomwerCIDatasets.get_dataset(
+                "edf_datasets/test01",
+            ),
+            dst=self.sourceT01,
+        )
         self.S3XMLFile = os.path.join(self.sourceS3, "scan_3_.xml")
         self.S3Ref0000 = os.path.join(self.sourceS3, "ref0000_0000.edf")
         self.S3Ref0010 = os.path.join(self.sourceS3, "ref0000_0010.edf")
@@ -61,34 +68,42 @@ class TestGetClosestEnergy(unittest.TestCase):
     def testEnergyFromEDF(self):
         os.remove(self.S3XMLFile)
         self.assertTrue(
-            utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0000) == 61
+            spec_utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0000)
+            == 61
         )
         self.assertTrue(
-            utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0010) == 61
+            spec_utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0010)
+            == 61
         )
 
     def testEnergyFromXML(self):
         os.remove(self.S3Ref0000)
         os.remove(self.S3Ref0010)
         self.assertTrue(
-            utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0000) == 10
+            spec_utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0000)
+            == 10
         )
         self.assertTrue(
-            utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0010) == 10
+            spec_utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0010)
+            == 10
         )
 
     def testEnergyFromInfo(self):
-        self.assertTrue(utils.getClosestEnergy(scan=self.sourceT01, refFile=None) == 19)
+        self.assertTrue(
+            spec_utils.getClosestEnergy(scan=self.sourceT01, refFile=None) == 19
+        )
 
     def testDefaultEnergy(self):
         os.remove(self.S3XMLFile)
         os.remove(self.S3Ref0000)
         os.remove(self.S3Ref0010)
         self.assertTrue(
-            utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0000) is None
+            spec_utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0000)
+            is None
         )
         self.assertTrue(
-            utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0010) is None
+            spec_utils.getClosestEnergy(scan=self.sourceS3, refFile=self.S3Ref0010)
+            is None
         )
 
 
@@ -96,7 +111,9 @@ class TestGetClosestSREnergy(unittest.TestCase):
     def setUp(self):
         self.topSrcFolder = tempfile.mkdtemp()
         self.dataSetID = "test10"
-        self.dataDir = UtilsTest.getEDFDataset(self.dataSetID)
+        self.dataDir = TomwerCIDatasets.get_dataset(
+            f"edf_datasets/{self.dataSetID}",
+        )
         self.sourceT10 = os.path.join(self.topSrcFolder, self.dataSetID)
         shutil.copytree(src=os.path.join(self.dataDir), dst=self.sourceT10)
         self.T10XMLFile = os.path.join(self.sourceT10, "test10.xml")
@@ -106,16 +123,6 @@ class TestGetClosestSREnergy(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree(self.topSrcFolder)
 
-    def testIntenistyFromInfo(self):
-        self.assertTrue(
-            utils.getClosestSRCurrent(scan_dir=self.sourceT10, refFile=None) == 101.3
-        )
-
-    def testDefaultIntensity(self):
-        os.remove(self.T10XMLFile)
-        os.remove(self.T10InfoFile)
-        self.assertTrue(utils.getClosestSRCurrent(scan_dir=self.sourceT10) is None)
-
 
 class TestMockEDFScan(unittest.TestCase):
     """Test that mock scan are adapted to other unit test"""
@@ -254,8 +261,8 @@ class TestMockEDFScan(unittest.TestCase):
             self.assertEqual(scan_dynamic[angle_id].file_path(), file_name)
 
 
-class TestMockHDF5(unittest.TestCase):
-    """Test MockHDF5 to check that the file produced is valid"""
+class TestMockNXtomo(unittest.TestCase):
+    """Test MockNXtomo to check that the file produced is valid"""
 
     def setUp(self) -> None:
         self._folder = tempfile.mkdtemp()
@@ -266,7 +273,7 @@ class TestMockHDF5(unittest.TestCase):
     def testSimpleMockCreationOneCall(self):
         """Test mock of an acquisition starting by one dark, then 10 ref,
         then 20 radios, then 10 'final' ref and 2 alignment radio"""
-        mock = MockHDF5(
+        mock = MockNXtomo(
             scan_path=self._folder,
             n_proj=20,
             n_ini_proj=20,