tomwer 1.2.1__py3-none-any.whl → 1.3.12__py3-none-any.whl

tomwer/gui/visualization/volumeviewer.py

@@ -1,35 +1,4 @@
-# coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
-
-__authors__ = ["H. Payno"]
-__license__ = "MIT"
-__date__ = "05/08/2020"
-
-
 import logging
-import os
 import weakref
 import h5py
 from typing import Union
@@ -46,11 +15,10 @@ from tomoscan.esrf.volume.hdf5volume import HDF5Volume
 from tomoscan.factory import Factory
 from tomoscan.identifier import VolumeIdentifier
 from tomoscan.volumebase import VolumeBase
-from tomoscan.io import HDF5File
 
 from tomwer.core.scan.scanbase import TomwerScanBase
-from tomwer.core.utils.ftseriesutils import get_vol_file_shape
 from tomwer.gui.visualization.reconstructionparameters import ReconstructionParameters
+from silx.gui.widgets.WaitingOverlay import WaitingOverlay
 
 _logger = logging.getLogger(__name__)
 
@@ -247,6 +215,9 @@ class _TomoApplicationContext(DataViewHooks):
 class VolumeViewer(qt.QMainWindow):
     def __init__(self, parent):
         qt.QMainWindow.__init__(self, parent)
+
+        self._volume_loaded_in_background = (None, None)
+        # store the volume loaded in the background and the thread used for it as (volume_identifier, thread)
         self._centralWidget = DataViewerFrame(parent=self)
         self.__context = _TomoApplicationContext(self)
         self._centralWidget.setGlobalHooks(self.__context)
@@ -254,6 +225,12 @@ class VolumeViewer(qt.QMainWindow):
             qt.QSizePolicy.Expanding, qt.QSizePolicy.Expanding
         )
         self.setCentralWidget(self._centralWidget)
+        # waiter overlay to notify user loading is on-going
+        self._waitingOverlay = WaitingOverlay(self._centralWidget)
+        self._waitingOverlay.setIconSize(qt.QSize(30, 30))
+        self._waitingOverlay.hide()
+
+        # display scan information when possible
         self._infoWidget = _ScanInfo(parent=self)
 
         # top level dock widget to display information regarding the scan
@@ -278,7 +255,6 @@ class VolumeViewer(qt.QMainWindow):
         """pointer to the hdf5 file since we want to set the HDF5Dataset for
         loading data on the fly and avoid loading everything into memory for
         hdf5."""
-        self.__first_load = True
         self.__last_mode = None
 
     def _close_h5_file(self):
@@ -308,9 +284,12 @@ class VolumeViewer(qt.QMainWindow):
         if volume is None:
             return
         self._set_volumes(volumes=(volume,))
-        # TODO in the future: do the equivalent of setScan from the volume metadata
 
     def _get_data_volume(self, volume: VolumeBase):
+        """
+        load the data of the requested volume.
+        :return: (data: Optional[str], state: str) state can be "loaded", "loading" or "failed"
+        """
         if not isinstance(volume, VolumeBase):
             raise TypeError(
                 f"volume is expected to be an instance of {VolumeBase}. Not {type(volume)}"
@@ -318,18 +297,73 @@ class VolumeViewer(qt.QMainWindow):
 
         self._close_h5_file()
         if isinstance(volume, HDF5Volume):
-            self._h5_file = HDF5File(filename=volume.data_url.file_path(), mode="r")
+            try:
+                self._h5_file = h5py.File(volume.data_url.file_path(), mode="r")
+            except OSError:
+                self._h5_file = h5py.File(
+                    volume.data_url.file_path(),
+                    mode="r",
+                    libver="latest",
+                    swmr=True,
+                )
+
             if volume.data_url.data_path() in self._h5_file:
                 data = self._h5_file[volume.data_url.data_path()]
+                state = "loaded"
             else:
                 data = None
+                state = "failed"
+        elif volume.data is not None:
+            data = volume.data
+            state = "loaded"
         else:
-            _logger.warning(
-                "Attempt to set a non HDF5 volume to the viewer. This requires to load all the data in memory. This can take a while"
-            )
-            data = volume.data if volume.data is not None else volume.load_data()
+            volume_id, _ = self._volume_loaded_in_background
+            if volume_id == volume.get_identifier().to_str():
+                # special case if the user send several time the volume which is currently loading
+                # in this case we just want to ignore the request to avoid reloading the volume
+                # can happen in the case of a large volume that take some time to be loaded.
+                pass
+            else:
+                _logger.warning(
+                    "Attempt to set a non HDF5 volume to the viewer. This requires to load all the data in memory. This can take a while"
+                )
+                self._stopExistingLoaderThread()
+                self._loadAndDisplay(volume)
+            state = "loading"
+            data = None
 
-        return data
+        return data, state
+
+    def _loaderThreadFinished(self):
+        """Callback activated when a VolumeLoader thread is finished"""
+        sender = self.sender()
+        if not isinstance(sender, VolumeLoader):
+            raise TypeError("sender is expected to be a VolumeLoader")
+
+        self._stopExistingLoaderThread()
+        if sender.volume.data is None:
+            _logger.error(f"Failed to load volume {sender.volume.get_identifier()}")
+        else:
+            self.setVolume(sender.volume)
+
+    def _loadAndDisplay(self, volume):
+        """Load a thread and add a callback when loading is done"""
+        loader_thread = VolumeLoader(volume=volume)
+        self._volume_loaded_in_background = (
+            volume.get_identifier().to_str(),
+            loader_thread,
+        )
+        loader_thread.finished.connect(self._loaderThreadFinished)
+        loader_thread.start()
+
+    def _stopExistingLoaderThread(self):
+        """Will stop any existing loader thread. Make sure we load one volume at most at the time"""
+        _, loader_thread = self._volume_loaded_in_background
+        if loader_thread is not None:
+            loader_thread.finished.disconnect(self._loaderThreadFinished)
+            if loader_thread.isRunning():
+                loader_thread.quit()
+            self._volume_loaded_in_background = (None, None)
 
     def _set_volumes(self, volumes: tuple):
         self.clear()
@@ -351,11 +385,7 @@ class VolumeViewer(qt.QMainWindow):
                     f"Volume should be an instance of a Volume, a VolumeIdentifier or a string refering to a VolumeIdentifier. {type(volume)} provided"
                 )
 
-            data = self._get_data_volume(volume)
-            # set volume dataset
-            if data is not None:
-                self._set_volume(data)
-            # set reconstruction parameters
+            # warning: load metadata before data because can get some conflict with the HDF5 reader flag if done after
             try:
                 # warning: limitation expected for .vol as it gets two configuration file. The default one is vol.info and does not contains
                 # any of the metadata 'distance', 'pixel size'... but it is here for backward compatiblity
@@ -367,81 +397,52 @@ class VolumeViewer(qt.QMainWindow):
                     f"Unable to set reconstruction parameters from {volume.data_url}. Not handled for pyhst reconstructions. Error is {e}"
                 )
 
+            data, state = self._get_data_volume(volume)
+            # set volume dataset
+            if state == "loading":
+                self._waitingOverlay.show()
+            elif state == "loaded":
+                self._waitingOverlay.hide()
+                if data is not None:
+                    self._set_volume(data)
+
+            elif state == "failed":
+                _logger.warning(
+                    f"Failed to load data from {volume.get_identifier().to_str()}"
+                )
+                return
+
     def _set_volume(self, volume: Union[numpy.ndarray, h5py.Dataset]):
         self._centralWidget.setData(volume)
-        if self.__first_load:
-            self._centralWidget.setDisplayMode(IMAGE_MODE)
-            self.__first_load = False
-        elif self.__last_mode is not None:
-            self._centralWidget.setDisplayMode(self.__last_mode)
+        self._centralWidget.setDisplayMode(self.__last_mode or IMAGE_MODE)
 
     def clear(self):
-        self.__last_mode = self._centralWidget.displayMode()
+        if self._centralWidget.displayMode():
+            self.__last_mode = self._centralWidget.displayMode()
         self._close_h5_file()
-        # self._centralWidget.setData(None)
         self._infoWidget.clear()
+        self._centralWidget.setData(None)
+        # if clear stop loading any volume
+        self._stopExistingLoaderThread()
 
     def sizeHint(self):
         return qt.QSize(600, 600)
 
-    # TODO: should be merged with dataviewer._load_vol function ?
-    def _load_vol(self, url):
-        """
-        load a .vol file
-        """
-        if url.file_path().lower().endswith(".vol.info"):
-            info_file = url.file_path()
-            raw_file = url.file_path().replace(".vol.info", ".vol")
-        else:
-            assert url.file_path().lower().endswith(".vol")
-            raw_file = url.file_path()
-            info_file = url.file_path().replace(".vol", ".vol.info")
 
-        if not os.path.exists(raw_file):
-            data = None
-            mess = f"Can't find raw data file {raw_file} associated with {info_file}"
-            _logger.warning(mess)
-        elif not os.path.exists(info_file):
-            mess = f"Can't find info file {info_file} associated with {raw_file}"
-            _logger.warning(mess)
-            data = None
-        else:
-            shape = get_vol_file_shape(info_file)
-            if None in shape:
-                _logger.warning(f"Fail to retrieve data shape for {info_file}.")
-                data = None
-            else:
-                try:
-                    numpy.zeros(shape)
-                except MemoryError:
-                    data = None
-                    _logger.warning(f"Raw file {raw_file} is too large for being read")
-                else:
-                    data = numpy.fromfile(
-                        raw_file, dtype=numpy.float32, count=-1, sep=""
-                    )
-                    try:
-                        data = data.reshape(shape)
-                    except ValueError:
-                        _logger.warning(
-                            f"unable to fix shape for raw file {raw_file}. "
-                            "Look for information in {info_file}"
-                        )
-                        try:
-                            sqr = int(numpy.sqrt(len(data)))
-                            shape = (1, sqr, sqr)
-                            data = data.reshape(shape)
-                        except ValueError:
-                            _logger.info(
-                                f"deduction of shape size for {raw_file} " "failed"
-                            )
-                            data = None
-                        else:
-                            _logger.warning(
-                                f"try deducing shape size for {raw_file} "
-                                "might be an incorrect interpretation"
-                            )
-        if url.data_slice() is None:
-            return data
-        else:
-            return data[url.data_slice()]
+class VolumeLoader(qt.QThread):
+    """
+    simple thread that load a volume in memory
+    """
+
+    def __init__(self, volume: VolumeBase) -> None:
+        super().__init__()
+        if not isinstance(volume, VolumeBase):
+            raise TypeError()
+        self.__volume = volume
+
+    def run(self):
+        self.__volume.load_data(store=True)
+
+    @property
+    def volume(self):
+        return self.__volume

tomwer/io/utils/h5pyutils.py

@@ -35,7 +35,7 @@ import contextlib
 
 import h5py
 from silx.io.url import DataUrl
-from tomoscan.io import HDF5File
+from silx.io.utils import open as open_hdf5
 
 
 class _BaseReader(contextlib.AbstractContextManager):
@@ -60,7 +60,7 @@ class EntryReader(_BaseReader):
     """Context manager used to read a bliss node"""
 
     def __enter__(self):
-        self._file_handler = HDF5File(filename=self._url.file_path(), mode="r")
+        self._file_handler = open_hdf5(filename=self._url.file_path())
         entry = self._file_handler[self._url.data_path()]
         if not isinstance(entry, h5py.Group):
             raise ValueError("Data path should point to a bliss node (h5py.Group)")
@@ -71,7 +71,7 @@ class DatasetReader(_BaseReader):
     """Context manager used to read a bliss node"""
 
     def __enter__(self):
-        self._file_handler = HDF5File(filename=self._url.file_path(), mode="r")
+        self._file_handler = open_hdf5(filename=self._url.file_path())
         entry = self._file_handler[self._url.data_path()]
         if not isinstance(entry, h5py.Dataset):
             raise ValueError("Data path should point to a dtaset (h5py.Dataset)")

tomwer/io/utils/raw_and_processed_data.py

@@ -0,0 +1,84 @@
+"""some utils to move a path to process_data or to raw_data"""
+
+import os
+from nxtomomill.converter.hdf5.utils import (
+    RAW_DATA_DIR_NAME,
+    PROCESSED_DATA_DIR_NAME,
+)
+
+
+def to_raw_data_path(file_path: str):
+    """convert a path to 'RAW_DATA' when possible"""
+    split_path = file_path.split(os.sep)
+    # reverse it to find the lower level value of 'PROCESSED_DATA_DIR_NAME' if by any 'chance' has several in the path
+    # in this case this is most likely what we want
+    split_path = split_path[::-1]
+    # check if already contain in a "RAW_DATA_DIR_NAME" directory
+    try:
+        index_raw_data = split_path.index(RAW_DATA_DIR_NAME)
+    except ValueError:
+        index_raw_data = None
+
+    try:
+        index_processed_data = split_path.index(PROCESSED_DATA_DIR_NAME)
+    except ValueError:
+        # if the value is not in the list
+        pass
+    else:
+        if index_raw_data is None or index_raw_data > index_processed_data:
+            # make sure we are not already in a 'PROCESSED_DATA' directory. Not sure it will never happen but safer
+            split_path[index_processed_data] = RAW_DATA_DIR_NAME
+
+    # reorder path to original
+    split_path = split_path[::-1]
+    return os.sep.join(split_path)
+
+
+def to_processed_data_path(file_path: str):
+    """convert a path to 'PROCESSED_DATA' when possible"""
+    split_path = file_path.split(os.sep)
+    # reverse it to find the lower level value of '_RAW_DATA_DIR_NAME' if by any 'chance' has several in the path
+    # in this case this is most likely what we want
+    split_path = split_path[::-1]
+    # check if already contain in a "PROCESSED_DATA" directory
+    try:
+        index_processed_data = split_path.index(PROCESSED_DATA_DIR_NAME)
+    except ValueError:
+        index_processed_data = None
+
+    try:
+        index_raw_data = split_path.index(RAW_DATA_DIR_NAME)
+    except ValueError:
+        # if the value is not in the list
+        pass
+    else:
+        if index_processed_data is None or index_raw_data < index_processed_data:
+            # make sure we are not already in a 'PROCESSED_DATA' directory. Not sure it will never happen but safer
+            split_path[index_raw_data] = PROCESSED_DATA_DIR_NAME
+
+    # reorder path to original
+    split_path = split_path[::-1]
+    return os.sep.join(split_path)
+
+
+def file_is_on_processed_data(file_path: str):
+    """Util that raises an error if path is not in PROCESSED_DATA folder"""
+    split_path = file_path.split(os.sep)
+    # reverse it to find the lower level value of 'PROCESSED_DATA_DIR_NAME' if by any 'chance' has several in the path
+    # in this case this is most likely what we want
+    split_path = split_path[::-1]
+    # check if already contain in a "RAW_DATA_DIR_NAME" directory
+    try:
+        index_raw_data = split_path.index(RAW_DATA_DIR_NAME)
+    except ValueError:
+        index_raw_data = None
+
+    try:
+        index_processed_data = split_path.index(PROCESSED_DATA_DIR_NAME)
+    except ValueError:
+        # if the value is not in the list
+        index_processed_data = None
+    else:
+        if index_raw_data is not None and index_raw_data < index_processed_data:
+            return False
+    return index_processed_data is not None

tomwer/io/utils/tomoobj.py

@@ -36,7 +36,7 @@ def get_tomo_objs_instances(tomo_objs: tuple) -> tuple:
     """
     instances = []
     has_scans = False
-    has_vols = True
+    has_vols = False
     for tomo_obj in tomo_objs:
 
         def get_scans():
@@ -45,9 +45,7 @@ def get_tomo_objs_instances(tomo_objs: tuple) -> tuple:
             except Exception:
                 try:
                     return ScanFactory.create_scan_objects(tomo_obj)
-                except Exception as e:
-                    raise e
-                    print("error is", e)
+                except Exception:
                     return tuple()
 
         def get_volumes():
@@ -68,12 +66,12 @@ def get_tomo_objs_instances(tomo_objs: tuple) -> tuple:
             # we start by scan because they have a 'definition' as NXtomo available.
             # otherwise we might get some entries defined as scan AND volume...
             # for stitching we expect users to ask of for one or the other.
-            volumes = get_volumes()
+            volumes = get_volumes() or ()
             if len(volumes) > 0:
                 has_vols = True
                 instances.extend(volumes)
         else:
-            has_scans = True
+            has_scans = len(scans_found) > 0
             instances.extend(scans_found)
 
     return tuple(instances), (has_scans, has_vols)

tomwer/io/utils/utils.py

@@ -37,9 +37,9 @@ import os
 import h5py
 import numpy.lib.npyio
 from PIL import Image
+from silx.io.utils import open as open_hdf5
 from tomoscan.esrf import has_glymur
 from tomoscan.esrf.scan.utils import get_data as tomoscan_get_data
-from tomoscan.io import HDF5File
 from typing import Union
 
 from tomwer.core.utils import ftseriesutils
@@ -190,7 +190,7 @@ def get_linked_files_with_entry(hdf5_file: str, entry: str) -> set:
     final_datasets = set()  # file_path, dataset_path
     treated_items = set()  # abs_file_path, data_path
 
-    abs_hdf5_file = os.path.realpath(hdf5_file)
+    abs_hdf5_file = os.path.abspath(hdf5_file)
     items_to_treat.add((abs_hdf5_file, hdf5_file, entry))
 
     while len(items_to_treat) > 0:
@@ -201,14 +201,14 @@ def get_linked_files_with_entry(hdf5_file: str, entry: str) -> set:
             if item in treated_items:
                 continue
             dirname = os.path.dirname(abs_file_path)
-            with HDF5File(abs_file_path, mode="r") as h5f:
+            with open_hdf5(abs_file_path) as h5f:
                 node = h5f.get(data_path, getlink=True)
                 if isinstance(node, h5py.ExternalLink):
                     ext_file_path = node.filename
                     if not os.path.isabs(ext_file_path):
                         ext_file_path = os.path.join(dirname, ext_file_path)
                     items_to_treat.add(
-                        (os.path.realpath(ext_file_path), node.filename, node.path)
+                        (os.path.abspath(ext_file_path), node.filename, node.path)
                     )
                 node = h5f.get(data_path, getlink=False)
                 if isinstance(node, h5py.Dataset) and node.is_virtual:
@@ -241,7 +241,7 @@ def get_linked_files_with_vds(hdf5_file: str, dataset_path: str) -> set:
     final_datasets = set()  # file_path, dataset_path
     treated_items = set()  # abs_file_path, dataset_path
 
-    abs_hdf5_file = os.path.realpath(hdf5_file)
+    abs_hdf5_file = os.path.abspath(hdf5_file)
     items_to_treat.add((abs_hdf5_file, hdf5_file, dataset_path))
 
     while len(items_to_treat) > 0:
@@ -252,7 +252,7 @@ def get_linked_files_with_vds(hdf5_file: str, dataset_path: str) -> set:
             if item in treated_items:
                 continue
             dirname = os.path.dirname(abs_file_path)
-            with HDF5File(abs_file_path, mode="r") as h5f:
+            with open_hdf5(abs_file_path) as h5f:
                 dataset = h5f[dataset_path]
                 if dataset.is_virtual:
                     for vs_info in dataset.virtual_sources():
@@ -261,7 +261,7 @@ def get_linked_files_with_vds(hdf5_file: str, dataset_path: str) -> set:
                             vs_file_path = os.path.join(dirname, vs_file_path)
                         items_to_treat.add(
                             (
-                                os.path.realpath(vs_file_path),
+                                os.path.abspath(vs_file_path),
                                 vs_info.file_name,
                                 vs_info.dset_name,
                             )

tomwer/resources/gui/icons/parameters.svg

@@ -14,7 +14,7 @@
    id="svg2985"
    version="1.1"
    inkscape:version="0.48.5 r10040"
-   sodipodi:docname="lamino_parameters.svg"
+   sodipodi:docname="parameters.svg"
    inkscape:export-filename="/nobackup/linazimov/payno/dev/esrf/ID06/id06workflow/resources/gui/icons/input.png"
    inkscape:export-xdpi="200"
    inkscape:export-ydpi="200">