tomwer 1.2.0a1__py3-none-any.whl → 1.2.0a3__py3-none-any.whl

tomwer/synctools/stacks/reconstruction/lamino.py

@@ -37,7 +37,7 @@ from queue import Queue
 from processview.core.superviseprocess import SuperviseProcess
 from silx.gui import qt
 
-from tomwer.core.process.reconstruction.lamino.tofu import LaminoReconstruction
+from tomwer.core.process.reconstruction.lamino.tofu import LaminoReconstructionTask
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.utils.Singleton import singleton
 
@@ -92,7 +92,7 @@ class LaminoReconstructionStack(qt.QObject, Queue):
                                      reconstruction
         :param callback: function to call after the reconstruction execution
         """
-        assert isinstance(recons_obj, LaminoReconstruction)
+        assert isinstance(recons_obj, LaminoReconstructionTask)
         Queue.put(
             self,
             (
@@ -199,7 +199,7 @@ class _LaminoReconsThread(qt.QThread, SuperviseProcess):
     def init(
         self, recons_obj, recons_params, additional_opts, remove_existing, scan_id
     ):
-        assert isinstance(recons_obj, LaminoReconstruction)
+        assert isinstance(recons_obj, LaminoReconstructionTask)
         self.recons_obj = recons_obj
         self.recons_params = recons_params
         self.additional_opts = additional_opts

tomwer/synctools/stacks/reconstruction/nabu.py

@@ -24,22 +24,21 @@
 
 __authors__ = ["H.Payno"]
 __license__ = "MIT"
-__date__ = "28/08/2020"
+__date__ = "17/07/2023"
 
 
-import gc
 import logging
 
 from processview.core.manager import DatasetState, ProcessManager
 from processview.core.superviseprocess import SuperviseProcess
 from silx.gui import qt
+from silx.gui.utils import blockSignals
 
-from tomwer.core.process.reconstruction.nabu import nabuslices, nabuvolume
-from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.settings import get_lbsram_path, isOnLbsram
 from tomwer.core.utils import isLowOnMemory
-from tomwer.io.utils import format_stderr_stdout
+from tomwer.core.process.reconstruction.nabu.nabuslices import NabuSlicesTask
+from tomwer.core.process.reconstruction.nabu.nabuvolume import NabuVolumeTask
 
 from ..processingstack import FIFO, ProcessingThread
 
@@ -93,6 +92,15 @@ class NabuSliceProcessStack(FIFO, qt.QObject):
     def setDryRun(self, dry_run):
         self._dry_run = dry_run
 
+    def cancel(self):
+        if self._computationThread.isRunning():
+            with blockSignals(self._computationThread):
+                self._computationThread.cancel()
+                # stop stack
+                super().stop()
+                # emit next
+                self.sigComputationEnded.emit(None, None)
+
 
 class NabuVolumeProcessStack(NabuSliceProcessStack):
     """Implementation of the `.AxisProcess` but having a stack for treating
@@ -117,6 +125,11 @@ class _SliceProcessingThread(ProcessingThread, SuperviseProcess):
         self._future_tomo_obj = None
         self._configuration = None
         self._dry_run = False
+        self._current_processing = None
+        # processing currently runned. As we want to be able to cancel processing we need to keep a pointer on the
+        # 'activate' nabu processing to stop process if requested
+        self._task = None
+        # ewoks task for reconstruction
 
     @property
     def future_tomo_obj(self):
@@ -128,8 +141,11 @@ class _SliceProcessingThread(ProcessingThread, SuperviseProcess):
     def init(self, data, configuration):
         self._scan = data
         self._configuration = configuration
+        self._task = None
 
     def run(self):
+        # note: now rnu does a processing close to the 'run_slices_reconstruction' except that we keep a trace on the
+        # current nabu subprocess runned. The goal is to be able to cancel / stop processing when asked
         self.sigComputationStarted.emit()
         mess = f"Start nabu slice(s) reconstruction of {self._scan}"
         _logger.processStarted(mess)
@@ -140,76 +156,22 @@ class _SliceProcessingThread(ProcessingThread, SuperviseProcess):
             details=mess,
         )
 
-        # # loop is required for distributed since version 2021
-        try:
-            (
-                succeed,
-                stdouts,
-                stderrs,
-                configs,
-                self._future_tomo_obj,
-            ) = nabuslices.run_slices_reconstruction(
-                config=self._configuration,
-                scan=self._scan,
-                dry_run=self._dry_run,
-                process_id=self.process_id,
-            )
-        except Exception as e:
-            _logger.processFailed(
-                f"Fail to compute slices for {str(self._scan)}. Reason is {e}."
-            )
-            ProcessManager().notify_dataset_state(
-                dataset=self._scan,
-                process=self,
-                state=DatasetState.FAILED,
-                details=mess,
-            )
-            self._future_tomo_obj = None
-        else:
-            index = self._scan.pop_process_index()
-            if isinstance(self._scan, EDFTomoScan):
-                entry = None
-            else:
-                entry = (
-                    self._scan.entry
-                )  # hotfix / FIXME: otherwise can fail to open the file to
-                # register the process. Might be fix since using HDF5File
-            gc.collect()
-            try:
-                nabuslices.NabuSlices._register_process(
-                    process_file=self._scan.process_file,
-                    entry=entry,
-                    configuration=self._configuration,
-                    results={},
-                    process=nabuslices.NabuSlices,
-                    process_index=index,
-                    overwrite=True,
-                )
-            except Exception as e:
-                _logger.warning(f"Fail to register NabuSlices process. Reason is {e}")
-
-            # TODO: check output files with the tomoscan validator ?
-            if not succeed:
-                mess = f"Slices computation for {self._scan} failed."
-                state = DatasetState.FAILED
-                _logger.processFailed(mess)
-            else:
-                state = DatasetState.SUCCEED
-                mess = f"Slices computed for {self._scan}."
-                _logger.processSucceed(mess)
-            elmts = [
-                format_stderr_stdout(stderr=stderr, stdout=stdout)
-                for stderr, stdout, config in zip(stderrs, stdouts, configs)
-            ]
-            elmts.insert(0, mess)
-            details = "\n".join(elmts)
+        self._task = NabuSlicesTask(
+            process_id=self.process_id,
+            inputs={
+                "data": self._scan,
+                "nabu_params": self._configuration,
+                "dry_run": self._dry_run,
+                "serialize_output_data": False,
+            },
+        )
+        self._task.run()
+        self._future_tomo_obj = self._task.outputs.future_tomo_obj
 
-            ProcessManager().notify_dataset_state(
-                dataset=self._scan,
-                process=self,
-                state=state,
-                details=details,
-            )
+    def cancel(self):
+        if self._task is not None:
+            self._task.cancel()
+        self.quit()
 
 
 class _VolumeProcessingThread(_SliceProcessingThread):
@@ -228,69 +190,16 @@ class _VolumeProcessingThread(_SliceProcessingThread):
             details=mess,
         )
 
-        # loop is required for distributed since version 2021
-        try:
-            (
-                succeed,
-                stdouts,
-                stderrs,
-                configs,
-                self._future_tomo_obj,
-            ) = nabuvolume.run_volume_reconstruction(
-                config=self._configuration,
-                scan=self._scan,
-                dry_run=self._dry_run,
-                process_id=self.process_id,
-            )
-        except Exception as e:
-            self._future_tomo_obj = None
-            mess = f"Fail to compute volume for {e}. Reason is {e}"
-            _logger.processFailed(mess)
-            ProcessManager().notify_dataset_state(
-                dataset=self._scan,
-                process=self,
-                state=DatasetState.FAILED,
-                details=mess,
-            )
-        else:
-            index = self._scan.pop_process_index()
-            if isinstance(self._scan, EDFTomoScan):
-                entry = None
-            else:
-                entry = self._scan.entry
-            gc.collect()
-            try:
-                nabuvolume.NabuVolume._register_process(
-                    process_file=self._scan.process_file,
-                    entry=entry,
-                    configuration=self._configuration,
-                    results={},
-                    process=nabuvolume.NabuVolume,
-                    process_index=index,
-                    overwrite=True,
-                )
-            except Exception as e:
-                _logger.warning(f"Fail to register NabuVolume process. Reason is {e}")
-            elmts = [
-                format_stderr_stdout(stderr=stderr, stdout=stdout, config=config)
-                for stderr, stdout, config in zip(stderrs, stdouts, configs)
-            ]
-
-            if not succeed:
-                mess = f"Volume computed for {self._scan} failed."
-                _logger.processFailed(mess)
-                state = DatasetState.FAILED
-            else:
-                mess = f"Volume computed for {self._scan}."
-                _logger.processSucceed(mess)
-                state = DatasetState.SUCCEED
-
-            elmts.insert(0, mess)
-            details = "\n".join(elmts)
-
-            ProcessManager().notify_dataset_state(
-                dataset=self._scan,
-                process=self,
-                state=state,
-                details=details,
-            )
+        self._task = NabuVolumeTask(
+            process_id=self.process_id,
+            inputs={
+                "data": self._scan,
+                "nabu_params": self._scan.nabu_recons_params,
+                "nabu_extra_params": self._configuration,
+                "process_id": self.process_id,
+                "dry_run": self._dry_run,
+                "serialize_output_data": False,  # avoid spending time on serialization / deserialization
+            },
+        )
+        self._task.run()
+        self._future_tomo_obj = self._task.outputs.future_tomo_obj

tomwer/synctools/stacks/reconstruction/normalization.py

@@ -115,6 +115,7 @@ class _ProcessingThread(ProcessingThread, SuperviseProcess):
             inputs={
                 "data": self._scan,
                 "configuration": self._i_norm_params,
+                "serialize_output_data": False,
             },
         )
         try:

tomwer/synctools/stacks/reconstruction/saaxis.py

@@ -34,11 +34,10 @@ from processview.core.manager import DatasetState, ProcessManager
 from processview.core.superviseprocess import SuperviseProcess
 from silx.gui import qt
 
-from tomwer.core.process.reconstruction.saaxis import SAAxisProcess
+from tomwer.core.process.reconstruction.saaxis import SAAxisTask
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.settings import get_lbsram_path, isOnLbsram
 from tomwer.core.utils import isLowOnMemory
-from tomwer.io.utils import format_stderr_stdout
 from tomwer.synctools.axis import QAxisRP
 from tomwer.synctools.saaxis import QSAAxisParams
 
@@ -110,7 +109,6 @@ class SAAxisProcessStack(FIFO, qt.QObject):
         )
 
     def _end_threaded_computation(self, callback=None):
-        assert self._data_currently_computed is not None
         self._computationThread.finished.disconnect()
         if callback:
             callback()
@@ -138,8 +136,11 @@ class _ProcessingThread(ProcessingThread, SuperviseProcess):
         self._patch_process_fct = None
         """function pointer to know which function to call for the axis
         calculation"""
-        self.__patch = {}
-        """Used to patch some calculation method (for test purpose)"""
+        self._current_processing = None
+        # processing currently runned. As we want to be able to cancel processing we need to keep a pointer on the
+        # 'activate' nabu processing to stop process if requested
+        self._task = None
+        # ewoks task for reconstruction
 
     def set_dry_run(self, dry_run):
         self._dry_run = dry_run
@@ -147,7 +148,7 @@ class _ProcessingThread(ProcessingThread, SuperviseProcess):
     def patch_processing(self, process_fct):
         self._patch_process_fct = process_fct
 
-    def init(self, data, saaxis_params):
+    def init(self, data, saaxis_params: dict):
         self._scan = data
         self._saaxis_params = saaxis_params
 
@@ -158,39 +159,24 @@ class _ProcessingThread(ProcessingThread, SuperviseProcess):
             for cor in self._saaxis_params.cors:
                 scores[cor] = self._patch_process_fct(cor)
             self._scan.saaxis_params.scores = scores
-            SAAxisProcess.autofocus(scan=self._scan)
+            SAAxisTask.autofocus(scan=self._scan)
             self.center_of_rotation = self._scan.saaxis_params.autofocus
         else:
-            saaxis = SAAxisProcess(
+            self._task = SAAxisTask(
                 process_id=self.process_id,
                 inputs={
                     "dump_process": False,
                     "data": self._scan,
                     "dry_run": self._dry_run,
+                    "sa_axis_params": self._saaxis_params.to_dict(),
+                    "serialize_output_data": False,
                 },
             )
-            saaxis.set_configuration(self._saaxis_params)
-            # loop is required for distributed since version 2021
-            try:
-                saaxis.run()
-            except Exception as e:
-                _logger.error(str(e))
-                mess = f"SAaxis computation for {str(self._scan)} failed."
-                state = DatasetState.FAILED
-            else:
-                mess = f"SAaxis computation for {str(self._scan)} succeed."
-                state = DatasetState.WAIT_USER_VALIDATION
-                self.center_of_rotation = self._scan.saaxis_params.autofocus
-
-            nabu_logs = []
-            for std_err, std_out in zip(saaxis.std_errs, saaxis.std_outs):
-                nabu_logs.append(format_stderr_stdout(stdout=std_out, stderr=std_err))
-            self._nabu_log = nabu_logs
-            nabu_logs.insert(0, mess)
-            details = "\n".join(nabu_logs)
-            ProcessManager().notify_dataset_state(
-                dataset=self._scan,
-                process=self,
-                state=state,
-                details=details,
-            )
+
+            self._task.run()
+            self.center_of_rotation = self._task.outputs.best_cor
+
+    def cancel(self):
+        if self._task is not None:
+            self._task.cancel()
+        self.quit()

tomwer/synctools/stacks/reconstruction/sadeltabeta.py

@@ -35,12 +35,11 @@ from processview.core.superviseprocess import SuperviseProcess
 from silx.gui import qt
 
 from tomwer.core.process.reconstruction.sadeltabeta.sadeltabeta import (
-    SADeltaBetaProcess,
+    SADeltaBetaTask,
 )
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.settings import get_lbsram_path, isOnLbsram
 from tomwer.core.utils import isLowOnMemory
-from tomwer.io.utils import format_stderr_stdout
 from tomwer.synctools.axis import QAxisRP
 from tomwer.synctools.sadeltabeta import QSADeltaBetaParams
 
@@ -134,15 +133,15 @@ class _ProcessingThread(ProcessingThread, SuperviseProcess):
             ProcessingThread.__init__(self, process_id=process_id)
         except TypeError:
             ProcessingThread.__init__(self)
-        self.center_of_rotation = None
+        self.db_value = None
         self._dry_run = False
         self._scan = None
         self._sa_delta_beta_params = None
         self._patch_process_fct = None
         """function pointer to know which function to call for the axis
         calculation"""
-        self.__patch = {}
-        """Used to patch some calculation method (for test purpose)"""
+        self._task = None
+        # ewoks task for reconstruction
 
     def set_dry_run(self, dry_run):
         self._dry_run = dry_run
@@ -161,39 +160,24 @@ class _ProcessingThread(ProcessingThread, SuperviseProcess):
             for db in self._sa_delta_beta_params.delta_beta_values:
                 scores[db] = self._patch_process_fct(db)
             self._scan.sa_delta_beta_params.scores = scores
-            SADeltaBetaProcess.autofocus(scan=self._scan)
+            SADeltaBetaTask.autofocus(scan=self._scan)
             self.db_value = self._scan.sa_delta_beta_params.autofocus
         else:
-            sa_delta_beta = SADeltaBetaProcess(
+            self._task = SADeltaBetaTask(
                 inputs={
                     "dump_process": False,
                     "data": self._scan,
                     "dry_run": self._dry_run,
-                    "sa_delta_beta_params": self._sa_delta_beta_params,
+                    "sa_delta_beta_params": self._sa_delta_beta_params.to_dict(),
+                    "serialize_output_data": False,
                 },
                 process_id=self.process_id,
             )
-            # loop is required for distributed since version 2021
-            try:
-                sa_delta_beta.run()
-            except Exception as e:
-                _logger.error(str(e))
-                mess = f"SADeltaBeta computation for {self._scan} failed."
-                state = DatasetState.FAILED
-            else:
-                mess = f"SADeltaBeta computation for {self._scan} succeed."
-                state = DatasetState.WAIT_USER_VALIDATION
-                self.db_value = self._scan.sa_delta_beta_params.autofocus
-
-            nabu_logs = []
-            for std_err, std_out in zip(sa_delta_beta.std_errs, sa_delta_beta.std_outs):
-                nabu_logs.append(format_stderr_stdout(stdout=std_out, stderr=std_err))
-            self._nabu_log = nabu_logs
-            nabu_logs.insert(0, mess)
-            details = "\n".join(nabu_logs)
-            ProcessManager().notify_dataset_state(
-                dataset=self._scan,
-                process=self,
-                state=state,
-                details=details,
-            )
+
+            self._task.run()
+            self.db_value = self._task.outputs.best_db
+
+    def cancel(self):
+        if self._task is not None:
+            self._task.cancel()
+        self.quit()

tomwer/synctools/test/test_darkRefs.py

@@ -36,7 +36,7 @@ import fabio
 import numpy
 from silx.gui.utils.testutils import TestCaseQt
 
-from tomwer.core.process.reconstruction.darkref.darkrefs import DarkRefs
+from tomwer.core.process.reconstruction.darkref.darkrefs import DarkRefsTask
 from tomwer.core.process.reconstruction.darkref.params import DKRFRP, ReduceMethod
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.scanutils import MockEDF
@@ -78,12 +78,13 @@ class TestDarkRefsBehavior(TestCaseQt):
         self.recons_params.flat_calc_method = ReduceMethod.NONE
         self.recons_params.dark_calc_method = ReduceMethod.MEDIAN
 
-        dar_ref_process = DarkRefs(
+        dar_ref_process = DarkRefsTask(
             inputs={
                 "dark_ref_params": self.recons_params,
                 "force_sync": True,
                 "darkhst_prefix": "darkHST",
                 "data": self.dataset_folder,
+                "serialize_output_data": False,
             }
         )
         dar_ref_process.run()
@@ -113,12 +114,13 @@ class TestDarkRefsBehavior(TestCaseQt):
         self.recons_params.flat_calc_method = ReduceMethod.MEDIAN
         self.recons_params.dark_calc_method = ReduceMethod.NONE
 
-        dar_ref_process = DarkRefs(
+        dar_ref_process = DarkRefsTask(
             inputs={
                 "dark_ref_params": self.recons_params,
                 "force_sync": True,
                 "darkhst_prefix": "darkHST",
                 "data": self.dataset_folder,
+                "serialize_output_data": False,
             }
         )
 
@@ -162,10 +164,11 @@ class TestRefCalculationOneSerie(TestCaseQt):
         )
 
         self.recons_params = DKRFRP()
-        self.darkRef = DarkRefs(
+        self.darkRef = DarkRefsTask(
             inputs={
                 "dark_ref_params": self.recons_params,
                 "data": self.tmp_dir,
+                "serialize_output_data": False,
             }
         )
         self.darkRef.setForceSync(True)
@@ -220,11 +223,12 @@ class TestRefCalculationThreeSerie(TestCaseQt):
                 file_desc.write(os.path.join(self.tmp_dir, refFile))
 
         self.recons_params = DKRFRP()
-        self.darkRef = DarkRefs(
+        self.darkRef = DarkRefsTask(
             inputs={
                 "dark_ref_params": self.recons_params,
                 "data": self.tmp_dir,
                 "force_sync": True,
+                "serialize_output_data": False,
             }
         )
         self.recons_params.flat_pattern = "ref*.*[0-9]{3,4}_[0-9]{3,4}"
@@ -275,11 +279,12 @@ class TestDarkCalculationOneFrame(TestCaseQt):
         file_desc.write(os.path.join(self.tmp_dir, "darkend0000.edf"))
         assert len(os.listdir(self.tmp_dir)) is (1 + n_scan + n_info + n_xml)
         self.recons_params = DKRFRP()
-        self.darkRef = DarkRefs(
+        self.darkRef = DarkRefsTask(
             inputs={
                 "dark_ref_params": self.recons_params,
                 "force_sync": True,
                 "data": self.tmp_dir,
+                "serialize_output_data": False,
             }
         )
 
@@ -330,11 +335,12 @@ class TestDarkCalculation(TestCaseQt):
         file_desc.write(os.path.join(self.tmp_dir, "darkend0000.edf"))
         assert len(os.listdir(self.tmp_dir)) is (1 + n_scan + n_xml + n_info)
         self.recons_params = DKRFRP()
-        self.darkRef = DarkRefs(
+        self.darkRef = DarkRefsTask(
             inputs={
                 "data": self.tmp_dir,
                 "dark_ref_params": self.recons_params,
                 "force_sync": True,
+                "serialize_output_data": False,
             }
         )
 
@@ -399,10 +405,11 @@ class TestDarkAccumulation(TestCaseQt):
     def testComputation(self):
         """Test data bone8_1_ from id16b containing dark.edf of reference
         and darkend"""
-        dark_ref_process = DarkRefs(
+        dark_ref_process = DarkRefsTask(
             inputs={
                 "data": os.path.join(self.outputdir, self.dataset),
                 "force_sync": True,
+                "serialize_output_data": False,
             }
         )
 
@@ -440,12 +447,13 @@ class TestPCOTomo(TestCaseQt):
         self.dataset = "pcotomo_2x2scan_refdarkbeg_end_conti"
         self.copyDataset(self.dataset)
         self.recons_params.dark_calc_method = ReduceMethod.MEDIAN
-        dar_ref_process = DarkRefs(
+        dar_ref_process = DarkRefsTask(
             inputs={
                 "dark_ref_params": self.recons_params,
                 "data": self.scan,
                 "darkhst_prefix": "darkHST",
                 "force_sync": True,
+                "serialize_output_data": False,
             }
         )
         dar_ref_process.run()
@@ -454,12 +462,13 @@ class TestPCOTomo(TestCaseQt):
         self.dataset = "pcotomo_2x2scan_refdarkbeg_end_conti"
         self.copyDataset(self.dataset)
         self.recons_params.flat_calc_method = ReduceMethod.MEDIAN
-        dar_ref_process = DarkRefs(
+        dar_ref_process = DarkRefsTask(
             inputs={
                 "dark_ref_params": self.recons_params,
                 "data": self.scan,
                 "darkhst_prefix": "darkHST",
                 "force_sync": True,
+                "serialize_output_data": False,
             }
         )
         dar_ref_process.run()

tomwer/synctools/test/test_foldertransfer.py

@@ -131,7 +131,7 @@ class TestEDFDataTransfert(TestCaseQt):
                 "dest_dir": self.targettedir,
                 "noRsync": True,
                 "move": True,
-                "force": False,
+                "overwrite": False,
             }
         )
         folder_trans_process.run()
@@ -145,7 +145,7 @@ class TestEDFDataTransfert(TestCaseQt):
                 "dest_dir": self.targettedir,
                 "noRsync": True,
                 "move": True,
-                "force": False,
+                "overwrite": False,
             }
         )
         with self.assertRaises(shutil.Error):
@@ -159,7 +159,7 @@ class TestEDFDataTransfert(TestCaseQt):
                 "dest_dir": self.targettedir,
                 "noRsync": True,
                 "move": True,
-                "force": True,
+                "overwrite": True,
             }
         )
         folder_trans_process.run()
@@ -181,7 +181,7 @@ class TestEDFDataTransfert(TestCaseQt):
                 "dest_dir": self.targettedir,
                 "noRsync": True,
                 "move": False,
-                "force": False,
+                "overwrite": False,
             }
         )
         folder_trans_process.run()
@@ -198,7 +198,7 @@ class TestEDFDataTransfert(TestCaseQt):
                 "dest_dir": self.targettedir,
                 "noRsync": True,
                 "move": False,
-                "force": False,
+                "overwrite": False,
             }
         )
 
@@ -213,7 +213,7 @@ class TestEDFDataTransfert(TestCaseQt):
                 "dest_dir": self.targettedir,
                 "noRsync": True,
                 "move": False,
-                "force": True,
+                "overwrite": True,
             }
         )
         folder_trans_process.run()
@@ -374,7 +374,7 @@ class TestPreTransfert(unittest.TestCase):
                 "dest_dir": self.outputfolder,
                 "data": self.scan,
                 "move": False,
-                "force": True,
+                "overwrite": True,
                 "noRsync": True,
             }
         )