tomwer 1.2.0a2__tar.gz → 1.2.0a3__tar.gz

{tomwer-1.2.0a2/tomwer.egg-info → tomwer-1.2.0a3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tomwer
-Version: 1.2.0a2
+Version: 1.2.0a3
 Summary: "tomography workflow tools"
 Home-page: https://gitlab.esrf.fr/tomotools/tomwer
 Author: data analysis unit

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/orangecontrib/tomwer/widgets/control/SingleTomoObjOW.py

@@ -139,9 +139,6 @@ class SingleTomoObjOW(OWBaseWidget, openclass=True):
         if isinstance(data, TomwerScanBase):
             if data.reduced_darks not in (None, {}):
                 reduced_darks = data.reduced_darks
-                reduced_darks[
-                    "reduce_frames_name"
-                ] = f"darks from {data.get_identifier().short_description()}"
                 self.Outputs.reduced_darks.send(reduced_darks)
 
                 # we want to send those in relative position to have something generic. This is a convention for now
@@ -157,9 +154,6 @@ class SingleTomoObjOW(OWBaseWidget, openclass=True):
 
             if data.reduced_flats not in (None, {}):
                 reduced_flats = data.reduced_flats
-                reduced_flats[
-                    "reduce_frames_name"
-                ] = f"flats from {data.get_identifier().short_description()}"
                 self.Outputs.reduced_flats.send(reduced_flats)
 
                 # we want to send those in relative position to have something generic. This is a convention for now

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/orangecontrib/tomwer/widgets/reconstruction/DarkRefAndCopyOW.py

@@ -246,16 +246,20 @@ class DarkRefAndCopyOW(SuperviseOW, WidgetLongProcessing):
         self.sigScanReady.emit(scan)
         if scan.reduced_darks not in (None, {}):
             # we want to send those in relative position to have something generic. This is a convention for now
+            reduced_darks = scan.reduced_darks
+            reduced_darks.pop("reduce_frames_name", None)
             self.Outputs.reduced_darks.send(
                 tomoscan.esrf.scan.utils.from_absolute_reduced_frames_to_relative(
-                    reduced_frames=scan.reduced_darks, scan=scan
+                    reduced_frames=reduced_darks, scan=scan
                 )
             )
         if scan.reduced_flats not in (None, {}):
             # we want to send those in relative position to have something generic. This is a convention for now
+            reduced_flats = scan.reduced_flats
+            reduced_flats.pop("reduce_frames_name", None)
             self.Outputs.reduced_flats.send(
                 tomoscan.esrf.scan.utils.from_absolute_reduced_frames_to_relative(
-                    reduced_frames=scan.reduced_flats, scan=scan
+                    reduced_frames=reduced_flats, scan=scan
                 )
             )
         _logger.info(f"{scan} ended")

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/tomwer/app/canvas_launcher/mainwindow.py

@@ -61,9 +61,7 @@ class MainWindow(_MainWindow):
     HELPDESK_URL = "https://requests.esrf.fr/plugins/servlet/desk/portal/41"
 
     def __init__(self, *args, **kwargs):
-        print("create window")
         super().__init__(*args, **kwargs)
-        print("end super")
         self.process_supervisor_dock = DockWidget(
             self.tr("object supervisor"),
             self,
@@ -71,7 +69,6 @@ class MainWindow(_MainWindow):
             allowedAreas=qt.Qt.BottomDockWidgetArea,
             visible=self.show_processes_manager_action.isChecked(),
         )
-        print("continue")
 
         self.process_supervisor_dock.setWidget(ProcessManagerWindow(parent=None))
         self.process_supervisor_dock.visibilityChanged[bool].connect(

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/tomwer/core/process/reconstruction/nabu/nabucommon.py

@@ -327,6 +327,18 @@ class _NabuBaseReconstructor:
         else:
             raise ValueError(f"{self.target} is not recognized as a valid target")
 
+    @staticmethod
+    def _get_gpu_and_cpu_mem_fraction(config_to_dump: dict):
+        gpu_mem_fraction = config_to_dump.get("resources", {}).get(
+            "gpu_mem_fraction", config_to_dump.get("gpu_mem_fraction", 0.9)
+        )
+        assert gpu_mem_fraction <= 1
+        cpu_mem_fraction = config_to_dump.get("resources", {}).get(
+            "cpu_mem_fraction", config_to_dump.get("cpu_mem_fraction", 0.9)
+        )
+        assert cpu_mem_fraction <= 1
+        return gpu_mem_fraction, cpu_mem_fraction
+
     def _run_nabu_locally(
         self,
         conf_file: str,
@@ -346,8 +358,22 @@ class _NabuBaseReconstructor:
         """
         if not has_nabu:
             raise ImportError("Fail to import nabu")
+        assert isinstance(config_to_dump, dict)
+        gpu_mem_fraction, cpu_mem_fraction = self._get_gpu_and_cpu_mem_fraction(
+            config_to_dump
+        )
+
         command = " ".join(
-            ("python3", "-m", settings.NABU_FULL_FIELD_APP_PATH, conf_file)
+            (
+                "python3",
+                "-m",
+                settings.NABU_FULL_FIELD_APP_PATH,
+                conf_file,
+                "--gpu_mem_fraction",
+                str(gpu_mem_fraction),
+                "--cpu_mem_fraction",
+                str(cpu_mem_fraction),
+            )
         )
         _logger.info(f'call nabu from "{command}"')
 
@@ -422,13 +448,21 @@ class _NabuBaseReconstructor:
         project_name = project_name.replace(" ", "_")
         cluster_config["job_name"] = project_name
 
+        # extract gpu_mem_fraction and cpu_mem_fraction
+        assert isinstance(config_to_dump, dict)
+        gpu_mem_fraction, cpu_mem_fraction = self._get_gpu_and_cpu_mem_fraction(
+            config_to_dump
+        )
+
         # submit job
         script_name = get_slurm_script_name(prefix="nabu")
         # for now force job name
         cluster_config["job_name"] = f"tomwer-nabu {conf_file}"
         job = SBatchScriptJob(
             slurm_config=cluster_config,
-            script=(f"python3 -m {settings.NABU_FULL_FIELD_APP_PATH} {conf_file}",),
+            script=(
+                f"python3 -m {settings.NABU_FULL_FIELD_APP_PATH} {conf_file} --gpu_mem_fraction {gpu_mem_fraction} --cpu_mem_fraction {cpu_mem_fraction}",
+            ),
             script_path=os.path.join(self.scan.path, "slurm_scripts", script_name),
             clean_script=False,
             working_directory=self.scan.working_directory,
@@ -455,10 +489,12 @@ class _NabuBaseReconstructor:
         """Return a tuple a potential callback to be launch once the future is done"""
         return tuple()
 
-    def _treateOutputSliceConfig(self, config):
+    def _treateOutputSliceConfig(self, config) -> tuple:
         """
         - add or overwrite some parameters of the dictionary
         - create the output directory if does not exist
+
+        :return: (config: dict, nabu_cfg_folder: str)
         """
         # handle phase
         pag = False
@@ -492,16 +528,26 @@ class _NabuBaseReconstructor:
             location = config["output"].get("location", None)
             if location not in ("", None):
                 location = format_output_location(location, scan=self.scan)
-                # if user specify the location
-                if not os.path.isdir(config["output"]["location"]):
-                    os.makedirs(location)
+                location_cfg_files = location
             else:
                 # otherwise default location will be the data root level
                 location = self.scan.path
+                location_cfg_files = location
+                # TODO: if is a single file - append prefix
+                if config["output"].get("file_format") in (
+                    NabuOutputFileFormat.EDF.value,
+                    NabuOutputFileFormat.TIFF.value,
+                    NabuOutputFileFormat.JP2K.value,
+                ):  # if user specify the location
+                    location = "/".join([location, _file_name])
+
             # add reconstruction path to the list. scan `reconstruction_paths` register all the existing path where
             # reconstruction are saved in order to be able to browse them all
             self.scan.add_reconstruction_path(location)
             config["output"]["location"] = location
+        else:
+            # don't think this could ever happen
+            location_cfg_files = self.scan.path
         # handle preproc
         if "preproc" not in config:
             config["preproc"] = {}
@@ -514,14 +560,14 @@ class _NabuBaseReconstructor:
 
         extra_infos = self.scan.intensity_normalization.get_extra_infos()
 
-        nabu_cfg_folders = os.path.join(
-            config["output"]["location"], settings.NABU_CFG_FILE_FOLDER
+        nabu_cfg_folder = os.path.join(
+            location_cfg_files, settings.NABU_CFG_FILE_FOLDER
         )
-        os.makedirs(nabu_cfg_folders, exist_ok=True)
+        os.makedirs(nabu_cfg_folder, exist_ok=True)
 
         # configuration file and nabu_tomwer_serving_hatch must be in the same folder
         serving_hatch_file = os.path.join(
-            nabu_cfg_folders, settings.NABU_TOMWER_SERVING_HATCH
+            nabu_cfg_folder, settings.NABU_TOMWER_SERVING_HATCH
         )
 
         source = extra_infos.get("source", INormSource.NONE)
@@ -576,8 +622,7 @@ class _NabuBaseReconstructor:
                 )
         else:
             raise NotImplementedError(f"source type {source.value} is not handled")
-
-        return config
+        return config, nabu_cfg_folder
 
     def _get_file_basename_reconstruction(self, pag, db, ctf):
         """return created file base name"""

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/tomwer/core/process/reconstruction/nabu/nabuscores.py

@@ -223,6 +223,8 @@ class _Reconstructor(_NabuBaseReconstructor):
             if self._cancelled:
                 break
             config, conf_file = self.preprocess_config(deepcopy(config), var_value)
+            print("conf file is", conf_file)
+            print("confif output is", config.get("output"))
 
             # add some tomwer metadata and save the configuration
             # note: for now the section is ignored by nabu but shouldn't stay that way
@@ -336,8 +338,7 @@ class _Reconstructor(_NabuBaseReconstructor):
             config["output"]["location"],
             nabu_settings.NABU_CFG_FILE_FOLDER,
         )
-        if not os.path.exists(cfg_folder):
-            os.makedirs(cfg_folder)
+        os.makedirs(cfg_folder, exist_ok=True)
 
         conf_file = os.path.join(
             cfg_folder, file_prefix + nabu_settings.NABU_CONFIG_FILE_EXTENSION

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/tomwer/core/process/reconstruction/nabu/nabuslices.py

@@ -716,17 +716,15 @@ class SingleSliceRunner(_NabuBaseReconstructor):
             "steps_file": steps_file,
         }
 
-        config = self._treateOutputSliceConfig(config)
+        config, cfg_folder = self._treateOutputSliceConfig(config)
         # the policy is to save nabu .cfg file at the same location as the
         # force overwrite results
         if self.slice_index is not None:
             config["reconstruction"]["start_z"] = self.slice_index
             config["reconstruction"]["end_z"] = self.slice_index
-        cfg_folder = os.path.join(
-            config["output"]["location"], nabu_settings.NABU_CFG_FILE_FOLDER
-        )
-        if not os.path.exists(cfg_folder):
-            os.makedirs(cfg_folder)
+
+        if self.slice_index is not None:
+            os.makedirs(config["output"]["location"], exist_ok=True)
 
         name = (
             config["output"]["file_prefix"] + nabu_settings.NABU_CONFIG_FILE_EXTENSION
@@ -828,7 +826,7 @@ class SingleSliceRunner(_NabuBaseReconstructor):
                 return "_".join(
                     (
                         basename + "slice_pag",
-                        str(slice_index).zfill(4),
+                        str(slice_index).zfill(6),
                         "db" + str(db).zfill(4),
                     )
                 )
@@ -836,12 +834,12 @@ class SingleSliceRunner(_NabuBaseReconstructor):
                 return "_".join(
                     (
                         basename + "slice_ctf",
-                        str(slice_index).zfill(4),
+                        str(slice_index).zfill(6),
                         "db" + str(db).zfill(4),
                     )
                 )
             else:
-                return "_".join((basename + "slice", str(slice_index).zfill(4)))
+                return "_".join((basename + "slice", str(slice_index).zfill(6)))
 
     @docstring(_NabuBaseReconstructor)
     def _get_file_basename_reconstruction(self, pag, db, ctf):

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/tomwer/core/process/reconstruction/nabu/nabuvolume.py

@@ -224,7 +224,9 @@ class VolumeRunner(_NabuBaseReconstructor):
         # force overwrite results
         if "output" not in self.configuration:
             self.configuration["output"] = {}
-        config_slices = self._treateOutputConfig(self.configuration)
+        config_slices, cfg_folder = self._treateOutputConfig(self.configuration)
+
+        # force overwriting results
         config_slices["output"].update({"overwrite_results": 1})
 
         # check and clamp `start_z` and `end_z`
@@ -242,12 +244,6 @@ class VolumeRunner(_NabuBaseReconstructor):
                     value = -1
                 config_slices["reconstruction"][key] = value
 
-        cfg_folder = os.path.join(
-            config_slices["output"]["location"], settings.NABU_CFG_FILE_FOLDER
-        )
-        if not os.path.exists(cfg_folder):
-            os.makedirs(cfg_folder)
-
         name = (
             config_slices["output"]["file_prefix"] + settings.NABU_CONFIG_FILE_EXTENSION
         )
@@ -256,6 +252,9 @@ class VolumeRunner(_NabuBaseReconstructor):
         conf_file = os.path.join(cfg_folder, name)
         _logger.info(f"{self.scan}: create {conf_file}")
 
+        # make sure output location exists
+        os.makedirs(config_slices["output"]["location"], exist_ok=True)
+
         # add some tomwer metadata and save the configuration
         # note: for now the section is ignored by nabu but shouldn't stay that way
         with utils.TomwerInfo(config_slices) as config_to_dump:
@@ -317,7 +316,9 @@ class VolumeRunner(_NabuBaseReconstructor):
         :return: (nabu config dict, nabu extra options)
         """
         config = copy.deepcopy(config)
-        config = super()._treateOutputSliceConfig(config)
+        config, nabu_cfg_folder = super()._treateOutputSliceConfig(config)
+        os.makedirs(config["output"]["location"], exist_ok=True)
+
         # adapt config_s to specific volume treatment
         if "postproc" in config:
             config["postproc"] = config["postproc"]
@@ -329,7 +330,7 @@ class VolumeRunner(_NabuBaseReconstructor):
             config["reconstruction"]["end_z"] = config["end_z"]
             del config["end_z"]
 
-        return config
+        return config, nabu_cfg_folder
 
     @docstring(_NabuBaseReconstructor)
     def _get_file_basename_reconstruction(self, pag, db, ctf):

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/tomwer/core/process/reconstruction/nabu/utils.py

@@ -159,7 +159,7 @@ def get_recons_volume_identifier(
     if file_format in ("hdf5", "h5", "hdf"):
         if slice_index is not None:
             # case of a single hdf5 file
-            file_name = "_".join((file_prefix, str(slice_index).zfill(4)))
+            file_name = "_".join((file_prefix, str(slice_index).zfill(6)))
         else:
             file_name = file_prefix
         file_name = ".".join((file_name, file_format))
@@ -179,23 +179,13 @@ def get_recons_volume_identifier(
     elif file_format in ("vol", "raw"):
         if slice_index is not None:
             # case of a single hdf5 file
-            file_name = "_".join((file_prefix, str(slice_index).zfill(4)))
+            file_name = "_".join((file_prefix, str(slice_index).zfill(6)))
         else:
             file_name = file_prefix
-        # on 2022.3.x the location seems invalid for nabu. Add this second location to
-        # handle most of the case.
-        fix_nabu = os.path.join(file_prefix, file_name)
-
         file_name = ".".join((file_name, file_format))
         file_path = os.path.join(location, file_name)
 
-        file_name_fix_nabu = ".".join((fix_nabu, file_format))
-        file_path_fix_nabu = os.path.join(location, file_name_fix_nabu)
-
-        volumes = (
-            RawVolume(file_path=file_path_fix_nabu),
-            RawVolume(file_path=file_path),
-        )
+        volumes = (RawVolume(file_path=file_path),)
     elif file_format in ("jp2", "jp2k", "edf", "tiff"):
         if file_format in ("jp2k", "jp2"):
             constructor = JP2KVolume
@@ -205,31 +195,15 @@ def get_recons_volume_identifier(
             constructor = TIFFVolume
         else:
             raise NotImplementedError
-        output_extension = file_format
         basename = file_prefix
+        file_path = location
+        volumes = (
+            constructor(
+                folder=location,
+                volume_basename=basename,
+            ),
+        )
 
-        if slice_index is None and start_z is None and end_z is None:
-            # if this is a single slice
-            file_path = ".".join(
-                (
-                    os.path.join(location, file_prefix, file_prefix),
-                    output_extension,
-                )
-            )
-            # TODO: in the future we might want to provide data_slice according to z values
-            volumes = (
-                constructor(
-                    folder=os.path.join(location, file_prefix),
-                    volume_basename=basename,
-                ),
-            )
-        else:
-            volumes = (
-                constructor(
-                    folder=os.path.join(location, file_prefix),
-                    volume_basename=basename,
-                ),
-            )
     # case of the multitiff. Not handled by tomwer
     # elif file_format == "tiff":
     #     # for single frame tiff nabu uses another convention by saving it 'directly'.

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/tomwer/core/process/test/test_nabu.py

@@ -127,7 +127,7 @@ class TestNabuIni(unittest.TestCase):
         # check volume and slice .cfg files for nabu reconstruction are here
         for file_name in (
             os.path.basename(self.scan.path) + ".cfg",
-            os.path.basename(self.scan.path) + "slice_0001" + ".cfg",
+            os.path.basename(self.scan.path) + "slice_000001" + ".cfg",
         ):
             with self.subTest(file_name=file_name):
                 self.assertTrue(
@@ -151,7 +151,7 @@ class TestNabuIni(unittest.TestCase):
         # check volume and slice .cfg files for nabu reconstruction are here
         for file_name in (
             os.path.basename(self.scan.path) + "pag_db0100.cfg",
-            os.path.basename(self.scan.path) + "slice_pag_0001_db0100" + ".cfg",
+            os.path.basename(self.scan.path) + "slice_pag_000001_db0100" + ".cfg",
         ):
             with self.subTest(file_name=file_name):
                 self.assertTrue(
@@ -178,7 +178,7 @@ class TestNabuIni(unittest.TestCase):
             ini_files.append(
                 os.path.basename(self.scan.path)
                 + "slice_"
-                + str(slice_index).zfill(4)
+                + str(slice_index).zfill(6)
                 + ".cfg"
             )
 
@@ -222,7 +222,7 @@ class TestNabuIni(unittest.TestCase):
             # add slice ini file
             ini_files.append(
                 os.path.basename(self.scan.path)
-                + "slice_pag_0001_db"
+                + "slice_pag_000001_db"
                 + str(int(pag_db)).zfill(4)
                 + ".cfg"
             )
@@ -265,7 +265,7 @@ class TestNabuIni(unittest.TestCase):
                 ini_files.append(
                     os.path.basename(self.scan.path)
                     + "slice_pag_"
-                    + str(slice_index).zfill(4)
+                    + str(slice_index).zfill(6)
                     + "_db"
                     + str(pag_db).zfill(4)
                     + ".cfg"

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/tomwer/gui/visualization/reconstructionparameters.py

@@ -160,15 +160,23 @@ class ReconstructionParameters(qt.QWidget):
         # voxel size can be stored as pixel size (old version) or voxel size (new version)
         recons_params = metadata.get("processing_options", {}).get("reconstruction", {})
         voxel_size_cm = recons_params.get("voxel_size_cm", None)
+
         # now voxel size is expected to be a tuple of three elements
         if voxel_size_cm is not None:
             voxel_size_cm = voxel_size_cm[0]
+            # FIXME: load_ini seems to fail to remove some char like '(' or ')'... to be fixed or investigate
+            # simplest might be to filter those when dumping it to text file... ??? or to handle those at silx level
+            if isinstance(voxel_size_cm, str):
+                for char_to_ignore in (" ", "(", ")", "[", "]"):
+                    voxel_size_cm = voxel_size_cm.replace(char_to_ignore, "")
         else:
             # backward compatibility with old volume
             voxel_size_cm = recons_params.get("pixel_size_cm", None)
         if voxel_size_cm is not None:
             voxel_size_cm = f"{float(voxel_size_cm):.8}"
-        self._pixelSizeQLE.setText(voxel_size_cm if voxel_size_cm is not None else "")
+        self._pixelSizeQLE.setText(
+            str(voxel_size_cm) if voxel_size_cm is not None else ""
+        )
 
     def _setCor(self, metadata: dict):
         cor = (

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/tomwer/gui/visualization/volumeviewer.py

@@ -357,6 +357,8 @@ class VolumeViewer(qt.QMainWindow):
                 self._set_volume(data)
             # set reconstruction parameters
             try:
+                # warning: limitation expected for .vol as it gets two configuration file. The default one is vol.info and does not contains
+                # any of the metadata 'distance', 'pixel size'... but it is here for backward compatiblity
                 self._reconsWidget.setVolumeMetadata(
                     volume.metadata or volume.load_metadata()
                 )

{tomwer-1.2.0a2 → tomwer-1.2.0a3}/tomwer/version.py

@@ -79,7 +79,7 @@ MAJOR = 1
 MINOR = 2
 MICRO = 0
 RELEV = "alpha"  # <16
-SERIAL = 2  # <16
+SERIAL = 3  # <16
 
 date = __date__
 

{tomwer-1.2.0a2 → tomwer-1.2.0a3/tomwer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tomwer
-Version: 1.2.0a2
+Version: 1.2.0a3
 Summary: "tomography workflow tools"
 Home-page: https://gitlab.esrf.fr/tomotools/tomwer
 Author: data analysis unit