tomwer 1.0.4__py3-none-any.whl → 1.1.0__py3-none-any.whl

tomwer/core/scan/blissscan.py

@@ -51,6 +51,7 @@ except ImportError:
 from silx.io.utils import h5py_read_dataset
 import logging
 from tomoscan.io import HDF5File
+from typing import Optional
 
 
 _logger = logging.getLogger(__name__)
@@ -135,8 +136,19 @@ class BlissScan:
         return "@".join((str(entry), master_file))
 
     def _deduce_transfert_scan(self, output_dir):
-        new_master_file = os.path.join(output_dir, os.path.basename(self.master_file))
-        return BlissScan(master_file=new_master_file, entry=self.entry)
+        new_master_file = os.path.join(
+            output_dir,
+            os.path.basename(os.path.dirname(self.master_file)),
+            os.path.basename(self.master_file),
+        )
+        new_proposal_file = os.path.join(
+            output_dir, os.path.basename(self.proposal_file)
+        )
+        return BlissScan(
+            master_file=new_master_file,
+            proposal_file=new_proposal_file,
+            entry=self.entry,
+        )
 
     @staticmethod
     def is_bliss_file(file_path):
@@ -146,6 +158,29 @@ class BlissScan:
         # for now there is not real way to know if a scan is abort or not
         return False
 
+    def scan_dir_name(self) -> Optional[str]:
+        """for 'this/is/my/acquisition' returns 'acquisition'"""
+        if self.path is not None:
+            return self.path.split(os.sep)[-1]
+        else:
+            return None
+
+    def scan_basename(self) -> Optional[str]:
+        """for 'this/is/my/acquisition' returns 'acquisition'"""
+        if self.path is not None:
+            return self.path
+        else:
+            return None
+
+    def scan_parent_dir_basename(self) -> Optional[str]:
+        if self.path is not None:
+            try:
+                return os.path.dirname(self.path)
+            except Exception:
+                return None
+        else:
+            return None
+
     @staticmethod
     def is_bliss_valid_entry(file_path: str, entry: str):
         def is_pcotmo_tomo(current_title: str):

tomwer/core/scan/edfscan.py

@@ -76,6 +76,10 @@ class EDFTomoScanIdentifier(_EDFTomoScanIdentifier, DatasetIdentifier):
             ObjClass=EDFTomoScan,
         )
 
+    def long_description(self) -> str:
+        """used for processview header tooltip for now"""
+        return self.to_str()
+
 
 class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
     """
@@ -87,6 +91,8 @@ class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
     def __init__(self, scan, overwrite_proc_file=False, update=True):
         _tsEDFTomoScan.__init__(self, scan=scan)
         TomwerScanBase.__init__(self)
+        # register at least the 'default' working directory as a possible reconstruction path
+        self.add_reconstruction_path(self.path)
 
         self._dark = None
         self._process_file = self.get_process_file_name(self)
@@ -99,19 +105,19 @@ class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
                 # fabio can raise some empty file error when data is not on disk (can be the case for the datawatcher when acquisiton is on going)
                 pass
         try:
-            reduced_darks = self.load_reduced_darks()
+            reduced_darks, metadata = self.load_reduced_darks(return_info=True)
         except (KeyError, OSError, ValueError):
             # file or key does not exists
             pass
         else:
-            self.set_reduced_darks(reduced_darks)
+            self.set_reduced_darks(reduced_darks, darks_infos=metadata)
 
         try:
-            reduced_flats = self.load_reduced_flats()
+            reduced_flats, metadata = self.load_reduced_flats(return_info=True)
         except (KeyError, OSError, ValueError):
             pass
         else:
-            self.set_reduced_flats(reduced_flats)
+            self.set_reduced_flats(reduced_flats, flats_infos=metadata)
 
     @staticmethod
     def is_tomoscan_dir(
@@ -250,6 +256,10 @@ class EDFTomoScan(_tsEDFTomoScan, TomwerScanBase):
                                 indexStr = local_str.split("slice_pag_")[-1].split("_")[
                                     0
                                 ]
+                            elif "slice_ctf_" in local_str:
+                                indexStr = local_str.split("slice_ctf_")[-1].split("_")[
+                                    0
+                                ]
                             else:
                                 indexStr = local_str.split("slice_")[-1].split("_")[0]
                             if containsDigits(indexStr):

tomwer/core/scan/hdf5scan.py

@@ -84,6 +84,10 @@ class HDF5TomoScanIdentifier(_HDF5TomoScanIdentifier, DatasetIdentifier):
             object=HDF5TomoScan, hdf5_file=hdf5_file, entry=entry
         )
 
+    def long_description(self) -> str:
+        """used for processview header tooltip for now"""
+        return self.to_str()
+
 
 class HDF5TomoScan(_tsHDF5TomoScan, TomwerScanBase):
     """
@@ -105,25 +109,27 @@ class HDF5TomoScan(_tsHDF5TomoScan, TomwerScanBase):
     def __init__(self, scan, entry, index=None, overwrite_proc_file=False):
         TomwerScanBase.__init__(self)
         _tsHDF5TomoScan.__init__(self, scan=scan, entry=entry, index=index)
+        # register at least the 'default' working directory as a possible reconstruction path
+        self.add_reconstruction_path(self.path)
 
         self._reconstruction_urls = None
         self._projections_with_angles = None
         self._process_file = self.get_process_file_name(self)
         self._init_index_process_file(overwrite_proc_file=overwrite_proc_file)
         try:
-            reduced_darks = self.load_reduced_darks()
+            reduced_darks, metadata = self.load_reduced_darks(return_info=True)
         except (KeyError, OSError, ValueError):
             # file or key does not exists
             pass
         else:
-            self.set_reduced_darks(reduced_darks)
+            self.set_reduced_darks(reduced_darks, darks_infos=metadata)
 
         try:
-            reduced_flats = self.load_reduced_flats()
+            reduced_flats, metadata = self.load_reduced_flats(return_info=True)
         except (KeyError, OSError, ValueError):
             pass
         else:
-            self.set_reduced_flats(reduced_flats)
+            self.set_reduced_flats(reduced_flats, flats_infos=metadata)
 
     @property
     def working_directory(self):

tomwer/core/scan/scanbase.py

@@ -109,6 +109,7 @@ class TomwerScanBase(TomwerObject):
         "list of url related to latest slice reconstruction from nabu"
         self._latest_vol_reconstructions = []
         """list of url related to latest volume reconstruction from nabu"""
+        self._reconstruction_paths = set()
 
     def _clear_heavy_cache(self):
         """For scan for now we don't want to remove any information from the cache.
@@ -233,6 +234,13 @@ class TomwerScanBase(TomwerObject):
     def reconstructions(self, reconstructions):
         self._reconstructions = reconstructions
 
+    @property
+    def reconstruction_paths(self):
+        return self._reconstruction_paths
+
+    def add_reconstruction_path(self, path: str):
+        self._reconstruction_paths.add(path)
+
     @property
     def nabu_recons_params(self):
         return self._nabu_params
@@ -325,7 +333,7 @@ class TomwerScanBase(TomwerObject):
             from ..process.reconstruction.axis.anglemode import CorAngleMode
 
             _mode = CorAngleMode.from_value(mode)
-            if self.path is None:
+            if self.path is None:  # pylint: disable=E1101
                 return None, None
 
             radios_with_angle = self.projections_with_angle()
@@ -461,10 +469,13 @@ class TomwerScanBase(TomwerObject):
             data = json.load(desc)
         else:
             data = desc
-        if not (self.DICT_PATH_KEY in data and data[self.DICT_TYPE_KEY] == self._TYPE):
+        if not (
+            self.DICT_PATH_KEY in data  # pylint: disable=E1101
+            and data[self.DICT_TYPE_KEY] == self._TYPE  # pylint: disable=E1101
+        ):
             raise ValueError("Description is not an EDFScan json description")
 
-        assert self.DICT_PATH_KEY in data
+        assert self.DICT_PATH_KEY in data  # pylint: disable=E1101
         assert self._DICT_LAMINO_RP_KEY in data
         self.lamino_recons_params = data[self._DICT_LAMINO_RP_KEY]
         # load axis reconstruction parameters
@@ -516,10 +527,10 @@ class TomwerScanBase(TomwerObject):
         return (
             isinstance(other, self.__class__)
             or isinstance(self, other.__class__)
-            and self.type == other.type
+            and self.type == other.type  # pylint: disable=E1101
             and self.lamino_recons_params == other.lamino_recons_params
             and self.nabu_recons_params == other.nabu_recons_params
-            and self.path == other.path
+            and self.path == other.path  # pylint: disable=E1101
         )
 
     def get_sinogram(self, line, subsampling=1, norm_method=None, **kwargs):
@@ -599,23 +610,19 @@ class TomwerScanBase(TomwerObject):
         :return: list of urls that contains reconstruction from nabu
         :rtype: list
         """
-        if (self.path is None) or (not os.path.isdir(self.path)):
+        all_recons_urls = set()
+        recons_paths = self.reconstruction_paths
+        for path in recons_paths:
+            all_recons_urls.update(self.get_reconstructions_urls_from_path(path))
+        return all_recons_urls
+
+    def get_reconstructions_urls_from_path(self, path, check_url=False):
+        if path is None or not os.path.isdir(path):
             return []
-
-        results = []
-
-        def get_recons_url_from_folder(folder, check_url=False):
-            """
-            retrieve all the url from a folder.
-
-            :param str folder:
-            :param bool check_url: if True before adding an url try to open it
-                                   and insure is valid
-            :return:
-            """
+        else:
             res = set()
-            for f in os.listdir(folder):
-                current_path = os.path.join(folder, f)
+            for f in os.listdir(path):
+                current_path = os.path.join(path, f)
                 if os.path.isfile(current_path):
                     volume = _get_reconstructed_single_file_volume(
                         current_path, scan=self
@@ -638,23 +645,20 @@ class TomwerScanBase(TomwerObject):
                     pass
                 else:
                     res.update(
-                        get_recons_url_from_folder(
-                            folder=current_path, check_url=check_url
+                        self.get_reconstructions_urls_from_path(
+                            path=current_path, check_url=check_url
                         )
                     )
             return res
 
-        results.extend(get_recons_url_from_folder(self.path))
-        return results
-
     @property
     def latest_reconstructions(self):
-        """List of latest reconstructions"""
+        """List of latest slices reconstructions (as VolumeIdentifier) - single slice volume"""
         return self._latest_reconstructions
 
     @property
     def latest_vol_reconstructions(self):
-        """List of latest reconstructions"""
+        """List of latest volume reconstructions (as VolumeIdentifier)"""
         return self._latest_vol_reconstructions
 
     def clear_latest_reconstructions(self):
@@ -722,8 +726,10 @@ class TomwerScanBase(TomwerObject):
                     res[value.path()] = key
             return res
 
-        proj_inv_url_to_index = invert_dict(self.projections)
-        alig_inv_url_to_index = invert_dict(self.alignment_projections)
+        proj_inv_url_to_index = invert_dict(self.projections)  # pylint: disable=E1101
+        alig_inv_url_to_index = invert_dict(
+            self.alignment_projections  # pylint: disable=E1101
+        )
         if url.path() in proj_inv_url_to_index:
             return proj_inv_url_to_index[url.path()]
         elif url.path() in alig_inv_url_to_index:

tomwer/core/scan/scanfactory.py

@@ -36,13 +36,11 @@ from tomoscan.identifier import BaseIdentifier, ScanIdentifier, VolumeIdentifier
 from tomoscan.tomoobject import TomoObject
 from tomwer.utils import docstring
 from typing import Union
-from .scanbase import TomwerScanBase
 from .edfscan import EDFTomoScan, EDFTomoScanIdentifier
 from .hdf5scan import HDF5TomoScan, HDF5TomoScanIdentifier
 from .blissscan import BlissScan
 import os
 import glob
-import json
 import logging
 from urllib.parse import urlparse
 from tomoscan.esrf.identifier.url_utils import split_path
@@ -114,6 +112,7 @@ class ScanFactory(object):
         else:
             raise ValueError(f"Scheme {scheme} not recognized")
 
+    @staticmethod
     def create_scan_object(scan_path, entry=None, accept_bliss_scan=False):
         """
 
@@ -130,7 +129,7 @@ class ScanFactory(object):
             raise ValueError("'scan_path' should be provided")
         if entry is not None and not entry.startswith("/"):
             entry = "/" + entry
-        if ScanFactory.is_hdf5_tomo(scan_path):
+        if os.path.isfile(scan_path) and ScanFactory.is_hdf5_tomo(scan_path):
             valid_entries = _oHDF5TomoScan.get_valid_entries(scan_path)
             if entry is None:
                 if len(valid_entries) > 1:
@@ -167,7 +166,7 @@ class ScanFactory(object):
         scan_path = scan_path.rstrip(os.path.sep)
         if EDFTomoScan.is_tomoscan_dir(scan_path):
             return (EDFTomoScan(scan=scan_path),)
-        elif os.path.isfile(scan_path) and HDF5TomoScan.is_tomoscan_dir(scan_path):
+        elif HDF5TomoScan.is_tomoscan_dir(scan_path):
             scans = []
             master_file = HDF5TomoScan.get_master_file(scan_path=scan_path)
             entries = HDF5TomoScan.get_valid_entries(master_file)
@@ -254,16 +253,3 @@ class ScanFactory(object):
             return h5py.is_hdf5(scan_path)
         else:
             return HDF5TomoScan.directory_contains_scan(scan_path)
-
-    @staticmethod
-    def create_from_json(desc):
-        """Create a ScanBase instance from a json description"""
-        data = json.load(desc)
-
-        if TomwerScanBase._DICT_TYPE_KEY not in data:
-            raise ValueError("json not recognize")
-        elif data[TomwerScanBase._DICT_TYPE_KEY] == EDFTomoScan._TYPE:
-            scan = EDFTomoScan(scan=None).load_from_dict(data)
-            return scan
-        else:
-            raise ValueError("Type", data[TomwerScanBase.type], "is not managed")

tomwer/core/scan/test/test_h5.py

@@ -32,7 +32,7 @@ import unittest
 import shutil
 import os
 import tempfile
-from tomwer.test.utils import UtilsTest
+from tomwer.tests.utils import UtilsTest
 from tomwer.core.scan.hdf5scan import HDF5TomoScan
 from tomwer.core.process.reconstruction.darkref.darkrefs import DarkRefs
 from tomwer.core.utils.scanutils import MockHDF5

tomwer/core/scan/test/test_process_registration.py

@@ -141,14 +141,3 @@ class TestProcessRegistration(unittest.TestCase):
             self.assertEqual(nodes["/entry/tomwer_process_16"], 16)
             self.assertTrue("/entry/tomwer_process_1" in nodes)
             self.assertEqual(nodes["/entry/tomwer_process_1"], 1)
-
-
-def suite():
-    test_suite = unittest.TestSuite()
-    for ui in (TestProcessRegistration,):
-        test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
-    return test_suite
-
-
-if __name__ == "__main__":
-    unittest.main(defaultTest="suite")

tomwer/core/scan/test/test_scan.py

@@ -36,7 +36,7 @@ import contextlib
 from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.process.task import Task
-from tomwer.test.utils import UtilsTest
+from tomwer.tests.utils import UtilsTest
 from tomwer.core.utils.scanutils import MockEDF
 from tomwer.core.process.reconstruction.nabu import nabuslices
 import shutil
@@ -90,38 +90,40 @@ class TestScanValidatorFindNabuFiles(unittest.TestCase):
 
     def test(self) -> None:
         for slice_index in (None, 1, 15):
-            for pag_db in (None, 100, 500):
-                file_name = (
-                    nabuslices.SingleSliceRunner.get_file_basename_reconstruction(
-                        slice_index=slice_index,
-                        scan=self.scan,
-                        pag=pag_db is not None,
-                        db=pag_db,
-                    )
-                )
-                for file_extension in ("npy", "npz", "hdf5"):
-                    with self.subTest(
-                        slice_index=slice_index,
-                        pag_db=pag_db,
-                        file_extension=file_extension,
-                    ):
-                        file_path = os.path.join(
-                            self.scan.path, file_name + "." + file_extension
+            for delta_beta in (None, 100, 500):
+                for pag in (True, False):
+                    for ctf in (True, False):
+                        file_name = nabuslices.SingleSliceRunner.get_file_basename_reconstruction(
+                            slice_index=slice_index,
+                            scan=self.scan,
+                            pag=pag if delta_beta is not None else False,
+                            ctf=ctf if delta_beta is not None else False,
+                            db=delta_beta,
                         )
-
-                        # simple context Manager which create the file
-                        # and remove it when leave
-                        with TestScanValidatorFindNabuFiles.FileCreator(file_path):
-                            n_to_discover = 1
-                            if slice_index is None:
-                                n_to_discover = 0
-                            self.assertEqual(
-                                len(
-                                    self.scan.get_reconstructions_paths(self.scan.path)
-                                ),
-                                n_to_discover,
+                        print("file_name is", file_name)
+                        with self.subTest(
+                            slice_index=slice_index,
+                            pag_db=delta_beta,
+                        ):
+                            file_path = os.path.join(
+                                self.scan.path, file_name + ".hdf5"
                             )
 
+                            # simple context Manager which create the file
+                            # and remove it when leave
+                            with TestScanValidatorFindNabuFiles.FileCreator(file_path):
+                                n_to_discover = 1
+                                if slice_index is None:
+                                    n_to_discover = 0
+                                self.assertEqual(
+                                    len(
+                                        self.scan.get_reconstructions_paths(
+                                            self.scan.path
+                                        )
+                                    ),
+                                    n_to_discover,
+                                )
+
 
 class TestScanValidatorFindPyHSTFiles(unittest.TestCase):
     """Function testing the getReconstructionsPaths function is correctly

tomwer/core/settings.py

@@ -96,7 +96,7 @@ class SlurmSettings:
     MEMORY_PER_WORKER = 50  # memory in GB
     """Amount of memory per worker"""
 
-    PARTIION = "p9gpu"
+    PARTITION = "p9gpu"
     """Queue / partition to use"""
 
     DEFAULT_WALLTIME = "01:00:00"  # None if the default wall time
@@ -104,7 +104,7 @@ class SlurmSettings:
     N_GPUS_PER_WORKER = 1
     """number of gpu per worker"""
 
-    PYTHON_VENV = "/scisoft/tomotools_env/tomwer/activate.sh latest"
+    PYTHON_VENV = "/scisoft/tomotools/activate.sh stable"
     """Python executable to take. Useful if compute nodes have a different environment from the front end.
     """
 

tomwer/core/test/test_utils.py

@@ -34,7 +34,7 @@ import os
 import shutil
 from tomwer.core import utils
 from tomwer.core.utils.scanutils import MockEDF, MockHDF5
-from tomwer.test.utils import UtilsTest
+from tomwer.tests.utils import UtilsTest
 from tomwer.core.utils.normalization import flatFieldCorrection
 import numpy
 

tomwer/core/tomwer_object.py

@@ -6,6 +6,7 @@ class TomwerObject(Dataset):
 
     def __init__(self) -> None:
         super().__init__()
+        self._cast_volume = None
 
     def _clear_heavy_cache(self):
         """util function to clear some heavy object from the cache"""
@@ -13,3 +14,21 @@ class TomwerObject(Dataset):
 
     def clear_caches(self):
         pass
+
+    @property
+    def cast_volume(self):
+        # for now this is used as an east way to cache the identifier and provide it to the remaining of the orange canvas.
+        # but this is a wrong designa and should be removed at one point.
+        return self._cast_volume
+
+    @cast_volume.setter
+    def cast_volume(self, volume):
+        from tomwer.core.volume.volumebase import TomwerVolumeBase
+
+        if not isinstance(volume, TomwerVolumeBase):
+            from tomwer.core.volume.volumefactory import (
+                VolumeFactory,
+            )  # avoid cyclic import
+
+            volume = VolumeFactory.create_tomo_object_from_identifier(identifier=volume)
+        self._cast_volume = volume

tomwer/core/utils/__init__.py

@@ -338,50 +338,6 @@ def getDim1Dim2(scan):
     return d1, d2
 
 
-# TODO: move to scan module
-def getStartEndVoxels(scan):
-    _info_file = os.path.join(scan, os.path.basename(scan) + ".info")
-    sv_1 = _getInformation(
-        scan=scan,
-        refFile=_info_file,
-        information="START_VOXEL_1",
-        aliases=["projectionSize/ROW_BEG"],
-    )
-    ev_1 = _getInformation(
-        scan=scan,
-        refFile=_info_file,
-        information="END_VOXEL_1",
-        aliases=["projectionSize/ROW_END"],
-    )
-
-    sv_2 = _getInformation(
-        scan=scan,
-        refFile=_info_file,
-        information="START_VOXEL_2",
-        aliases=["projectionSize/COL_BEG"],
-    )
-    ev_2 = _getInformation(
-        scan=scan,
-        refFile=_info_file,
-        information="END_VOXEL_2",
-        aliases=["projectionSize/COL_END"],
-    )
-
-    sv_3 = _getInformation(scan=scan, refFile=_info_file, information="START_VOXEL_3")
-    ev_3 = _getInformation(scan=scan, refFile=_info_file, information="END_VOXEL_3")
-
-    dim_1, dim_2 = getDim1Dim2(scan=scan)
-    sv_1 = sv_1 or 0
-    sv_2 = sv_2 or 0
-    sv_3 = sv_3 or 0
-
-    ev_1 = ev_1 or dim_1 - 1
-    ev_2 = ev_2 or dim_1 - 1
-    ev_3 = ev_3 or dim_2 - 1
-
-    return (sv_1, ev_1, sv_2, ev_2, sv_3, ev_3)
-
-
 # TODO: move to scan module
 def getParametersFromParOrInfo(_file):
     """
@@ -461,7 +417,6 @@ def getPixelSize(scan):
 
 
 url_base = "http://www.edna-site.org/pub/tomwer/"
-# url_base = 'http://ftp.edna-site.org/tomwer/'
 
 
 def DownloadDataset(dataset, output_folder, timeout, unpack=False):

tomwer/core/utils/char.py

@@ -39,3 +39,5 @@ DELTA_CHAR = (b"\xce\xb4").decode("utf-8")
 PSI_CHAR = (b"\xcf\x88").decode("utf-8")
 
 THETA_CHAR = (b"\xce\xb8").decode("utf-8")
+
+MU_CHAR = (b"\xce\xbc").decode("utf-8")

tomwer/core/utils/gpu.py

@@ -35,12 +35,12 @@ _logger = logging.getLogger(__name__)
 
 def getNumberOfDevice():
     try:
-        import pycuda  # noqa F401
-        from pycuda import compiler  # noqa F401
-        import pycuda.driver as drv
+        import pycuda  # noqa F401  pylint: disable=E0401
+        from pycuda import compiler  # noqa F401  pylint: disable=E0401
+        from pycuda import driver  # pylint: disable=E0401
 
-        drv.init()
-        return drv.Device.count()
+        driver.init()
+        return driver.Device.count()
     except Exception:
         _logger.error("fail to discover the number of gpu")
         return None

tomwer/core/utils/nxtomoutils.py

@@ -29,7 +29,7 @@ __date__ = "02/12/2021"
 
 
 from typing import Union
-from tomoscan.esrf.hdf5scan import ImageKey
+from tomoscan.esrf.scan.hdf5scan import ImageKey
 import numpy
 
 
@@ -38,7 +38,7 @@ def get_n_series(image_key_values: Union[tuple, list], image_key_type: ImageKey)
     Return the number of series of an inage_key. Image key can be dark, flat, or projection.
     A serie is defined as a contiguous elements in image_key_values
 
-    :param image_key_values: list or tuple of image_keys to consider. Can be integers or tomoscan.esrf.hdf5scan.ImageKey
+    :param image_key_values: list or tuple of image_keys to consider. Can be integers or tomoscan.esrf.scan.hdf5scan.ImageKey
     """
     image_key_type = ImageKey.from_value(image_key_type)
     if image_key_type is ImageKey.INVALID: