tomwer 1.4.0rc5__py3-none-any.whl → 1.4.1__py3-none-any.whl

tomwer/core/process/reconstruction/params_cache.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from tomwer.core.scan.scanbase import TomwerScanBase
+from collections import OrderedDict
+
+_CACHE: {str, tuple[float | None, dict | None]} = OrderedDict()
+"""
+In the case we want to reprocess a scan we (might) need to know the cor and the nabu reconstruction parameters (for nabu slice and volume reconstructions)
+But we are not keeping all those object during the full 'orange canvas' lifecycle, it would be too heavy.
+As a consequence we cache those (small) values (and only those) of the last n scans (n==30) in case users want to reprocess some of those.
+n value is an arbitrary value.
+"""
+_CACHE_SIZE = 30
+
+
+def save_reconstruction_parameters_to_cache(scan: TomwerScanBase):
+    # pop if exists. Will move this item as 'last-in'
+    _CACHE.pop(scan.get_identifier().to_str(), None)
+    _CACHE[scan.get_identifier().to_str()] = (
+        None if scan.axis_params is None else scan.axis_params.relative_cor_value,
+        None if scan.nabu_recons_params is None else scan.nabu_recons_params,
+    )
+    if len(_CACHE) > _CACHE_SIZE:
+        _CACHE.popitem(last=False)
+
+
+def load_reconstruction_parameters_from_cache(scan: TomwerScanBase):
+    cor, nabu_recons_params = _CACHE.get(scan.get_identifier().to_str(), (None, None))
+    if cor is not None and scan.axis_params is not None:
+        scan.axis_params.set_relative_value(cor)
+    if nabu_recons_params is not None:
+        scan.nabu_recons_params = nabu_recons_params
+
+
+def clear_reconstruction_parameters_cache():
+    _CACHE.clear()

tomwer/core/process/reconstruction/saaxis/saaxis.py

@@ -5,15 +5,16 @@ from __future__ import annotations
 
 import logging
 import os
-
 import h5py
+
 import numpy
 from multiprocessing import Pool
 from processview.core.manager import DatasetState, ProcessManager
 from processview.core.superviseprocess import SuperviseProcess
 from silx.io.url import DataUrl
-from tomwer.core.utils.deprecation import deprecated_warning
+from silx.io.utils import h5py_read_dataset
 
+from tomwer.core.utils.deprecation import deprecated_warning
 from tomoscan.io import HDF5File
 
 import tomwer.version
@@ -33,13 +34,14 @@ from tomwer.core.process.reconstruction.scores import (
 from tomwer.core.process.reconstruction.scores.params import ScoreMethod
 from tomwer.core.process.reconstruction.utils.cor import relative_pos_to_absolute
 from tomwer.core.process.task import Task
-from tomwer.core.scan.nxtomoscan import NXtomoScan
+from tomwer.core.process import utils as core_utils
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils import logconfig
-from tomwer.core.utils.locker import FileLockerManager
 from tomwer.core.utils.scanutils import data_identifier_to_scan
 from tomwer.core.utils.slurm import is_slurm_available
+
+from tomwer.io.utils.h5pyutils import EntryReader
 from tomwer.core.volume.volumefactory import VolumeFactory
 from tomwer.io.utils.utils import get_slice_data
 from tomwer.io.utils import format_stderr_stdout
@@ -378,7 +380,8 @@ class SAAxisTask(
         output_dir = params.output_dir or None
         if output_dir is None:
             output_dir = (
-                params.nabu_params.get("output", {}).get("location", None) or None
+                params.nabu_recons_params.get("output", {}).get("location", None)
+                or None
             )
             if output_dir is None:
                 output_dir = os.path.join(scan.path, DEFAULT_RECONS_FOLDER)
@@ -576,35 +579,23 @@ class SAAxisTask(
                 _logger.error(e)
             else:
                 if self._dump_process:
-                    process_idx = SAAxisTask.process_to_tomwer_processes(
-                        scan=scan,
-                    )
-                    if self.dump_roi and process_idx is not None:
-                        self.dump_rois(
-                            scan, score_rois=score_rois, process_index=process_idx
-                        )
+                    self.save_results_to_disk(scan=scan)
+                    if self.dump_roi:
+                        self.dump_rois(scan, score_rois=score_rois)
 
     @staticmethod
-    def dump_rois(scan, score_rois, process_index):
-        process_file = scan.process_file
-        process_name = "tomwer_process_" + str(process_index)
-
+    def dump_rois(scan, score_rois):
         if scan.saaxis_params.scores in (None, {}):
             return
 
-        def get_process_path():
-            return "/".join((scan.entry or "entry", process_name))
+        process_url = SAAxisTask.get_results_url(scan=scan)
 
-        # save it to the file
-        with FileLockerManager.get_lock(process_file):
-            # needs an extra lock for multiprocessing
-
-            with HDF5File(process_file, mode="a") as h5f:
-                nx_process = h5f.require_group(get_process_path())
-                score_roi_grp = nx_process.require_group("score_roi")
-                for cor, roi in score_rois.items():
-                    score_roi_grp[str(cor)] = roi
-                    score_roi_grp[str(cor)].attrs["interpretation"] = "image"
+        with HDF5File(process_url.file_path(), mode="w") as h5f:
+            nx_process = h5f.require_group(process_url.data_path())
+            score_roi_grp = nx_process.require_group("score_roi")
+            for cor, roi in score_rois.items():
+                score_roi_grp[str(cor)] = roi
+                score_roi_grp[str(cor)].attrs["interpretation"] = "image"
 
     @staticmethod
     def program_name():
@@ -622,74 +613,75 @@ class SAAxisTask(
         return "Semi automatic center of rotation / axis calculation"
 
     @staticmethod
-    def process_to_tomwer_processes(scan):
-        if scan.process_file is not None:
-            entry = "entry"
-            if isinstance(scan, NXtomoScan):
-                entry = scan.entry
-
-            cor = None
-            if hasattr(scan, "axis_params"):
-                cor = scan.axis_params.relative_cor_value
-
-            process_index = scan.pop_process_index()
-            try:
-                with scan.acquire_process_file_lock():
-                    Task._register_process(
-                        process_file=scan.process_file,
-                        entry=entry,
-                        results={"center_of_rotation": cor if cor is not None else "-"},
-                        configuration=scan.saaxis_params.to_dict(),
-                        process_index=process_index,
-                        overwrite=True,
-                        process=SAAxisTask,
-                    )
-                    SAAxisTask._extends_results(
-                        scan=scan, entry=entry, process_index=process_index
-                    )
-            except Exception as e:
-                _logger.warning(
-                    f"Fail to register process of with index {process_index}. Reason is {e}"
-                )
-            return process_index
+    def get_results_url(scan):
+        return DataUrl(
+            file_path=scan.get_relative_file(
+                os.path.join(DEFAULT_RECONS_FOLDER, "tomwer_saaxis.h5")
+            ),
+            data_path=scan.entry or "entry",
+            scheme="silx",
+        )
 
     @staticmethod
-    def _extends_results(scan, entry, process_index):
-        process_file = scan.process_file
-        process_name = "tomwer_process_" + str(process_index)
-
+    def save_results_to_disk(scan):
         if scan.saaxis_params.scores in (None, {}):
             return
 
-        def get_process_path():
-            return "/".join((entry or "entry", process_name))
+        saaxis_results_url = SAAxisTask.get_results_url(scan=scan)
 
         # save it to the file
-        with FileLockerManager().get_lock(process_file):
-            # needs an extra lock for multiprocessing
-
-            with HDF5File(process_file, mode="a") as h5f:
-                nx_process = h5f.require_group(get_process_path())
-                if "NX_class" not in nx_process.attrs:
-                    nx_process.attrs["NX_class"] = "NXprocess"
-
-                results = nx_process.require_group("results")
-                for cor, (url, score) in scan.saaxis_params.scores.items():
-                    results_cor = results.require_group(str(cor))
-                    for method in ScoreMethod:
-                        method_score = score.get(method)
-                        if method_score is None:
-                            results_cor[method.value] = "None"
-                        else:
-                            results_cor[method.value] = method_score
-
-                    link_path = os.path.relpath(
-                        url.file_path(),
-                        os.path.dirname(process_file),
-                    )
-                    results_cor["reconstructed_slice"] = h5py.ExternalLink(
-                        link_path, url.data_path()
+        with HDF5File(saaxis_results_url.file_path(), mode="a") as h5f:
+            nx_process = h5f.require_group(saaxis_results_url.data_path())
+            if "NX_class" not in nx_process.attrs:
+                nx_process.attrs["NX_class"] = "NXprocess"
+
+            results = nx_process.require_group("results")
+            for cor, (url, score) in scan.saaxis_params.scores.items():
+                results_cor = results.require_group(str(cor))
+                for method in ScoreMethod:
+                    method_score = score.get(method)
+                    if method_score is None:
+                        results_cor[method.value] = "None"
+                    else:
+                        results_cor[method.value] = method_score
+
+                link_path = os.path.relpath(
+                    url.file_path(),
+                    os.path.dirname(saaxis_results_url.file_path()),
+                )
+                results_cor["reconstructed_slice"] = h5py.ExternalLink(
+                    link_path, url.data_path()
+                )
+
+    @staticmethod
+    def load_results_from_disk(scan):
+        saaxis_results_url = SAAxisTask.get_results_url(scan=scan)
+
+        if saaxis_results_url.file_path() is None or not os.path.exists(
+            saaxis_results_url.file_path()
+        ):
+            _logger.warning(
+                "Unable to find process file. Unable to read " "existing processing"
+            )
+            return None, None
+
+        try:
+            with EntryReader(saaxis_results_url) as h5f_entry_node:
+                scores = core_utils.get_scores(h5f_entry_node)
+                if (
+                    "results" in h5f_entry_node
+                    and "center_of_rotation" in h5f_entry_node["results"]
+                ):
+                    selected = h5py_read_dataset(
+                        h5f_entry_node["results"]["center_of_rotation"]
                     )
+                else:
+                    _logger.warning(f"no results found for {scan}")
+                    selected = None
+                return scores, selected
+        except ValueError:
+            _logger.warning(f"Data path ({saaxis_results_url.data_path()}) not found")
+            return None, None
 
     def cancel(self):
         """

tomwer/core/process/reconstruction/sadeltabeta/sadeltabeta.py

@@ -20,6 +20,7 @@ from processview.core.manager import DatasetState, ProcessManager
 from processview.core.superviseprocess import SuperviseProcess
 from tomoscan.esrf.scan.utils import get_data
 from tomoscan.io import HDF5File
+from silx.io.utils import h5py_read_dataset
 
 import tomwer.version
 from tomwer.core.process.reconstruction.nabu.nabucommon import (
@@ -36,21 +37,21 @@ from tomwer.core.process.reconstruction.scores import (
     compute_score,
     get_disk_mask_radius,
 )
-from tomwer.core.utils.locker import FileLockerManager
 from tomwer.core.process.task import Task
-from tomwer.core.scan.nxtomoscan import NXtomoScan
+from tomwer.core.process import utils as core_utils
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils import logconfig
 from tomwer.core.utils.scanutils import data_identifier_to_scan
 from tomwer.core.volume.volumefactory import VolumeFactory
 from tomwer.io.utils import format_stderr_stdout
+from tomwer.io.utils.h5pyutils import EntryReader
 from tomwer.core.process.reconstruction.nabu.nabuscores import (
     run_nabu_one_slice_several_config,
 )
 from tomwer.core.futureobject import FutureTomwerObject
 
-
+from silx.io.url import DataUrl
 from silx.utils.deprecation import deprecated, deprecated_warning
 
 from ..nabu import utils as nabu_utils
@@ -202,7 +203,8 @@ class SADeltaBetaTask(
 
         if params.output_dir is None:
             output_dir = (
-                params.nabu_params.get("output", {}).get("location", None) or None
+                params.nabu_recons_params.get("output", {}).get("location", None)
+                or None
             )
             if output_dir is None:
                 output_dir = os.path.join(scan.path, DEFAULT_RECONS_FOLDER)
@@ -587,39 +589,30 @@ class SADeltaBetaTask(
                 _logger.error(e)
             else:
                 if self._dump_process:
-                    process_idx = SADeltaBetaTask.process_to_tomwer_processes(
-                        scan=scan,
-                    )
-                    if self.dump_roi and process_idx is not None:
-                        self.dump_rois(
-                            scan, score_rois=score_rois, process_index=process_idx
-                        )
+                    self.save_results_to_disk(scan=scan)
+                    if self.dump_roi:
+                        self.dump_rois(scan, score_rois=score_rois)
 
     @staticmethod
-    def dump_rois(scan, score_rois: dict, process_index: int):
-        if score_rois is None or len(score_rois) == 0:
-            return
+    def dump_rois(scan, score_rois: dict):
         if not isinstance(score_rois, dict):
             raise TypeError("score_rois is expected to be a dict")
-        process_file = scan.process_file
-        process_name = f"tomwer_process_{process_index}"
+
+        if score_rois is None or len(score_rois) == 0:
+            return
 
         if scan.saaxis_params.scores in (None, {}):
             return
 
-        def get_process_path():
-            return "/".join((scan.entry or "entry", process_name))
+        sa_delta_beta_results_url = SADeltaBetaTask.get_results_url(scan=scan)
 
         # save it to the file
-        with FileLockerManager().get_lock(process_file):
-            # needs an extra lock for multiprocessing
-
-            with HDF5File(process_file, mode="a") as h5f:
-                nx_process = h5f.require_group(get_process_path())
-                score_roi_grp = nx_process.require_group("score_roi")
-                for db, roi in score_rois.items():
-                    score_roi_grp[str(db)] = roi
-                    score_roi_grp[str(db)].attrs["interpretation"] = "image"
+        with HDF5File(sa_delta_beta_results_url.file_path(), mode="a") as h5f:
+            nx_process = h5f.require_group(sa_delta_beta_results_url.data_path())
+            score_roi_grp = nx_process.require_group("score_roi")
+            for db, roi in score_rois.items():
+                score_roi_grp[str(db)] = roi
+                score_roi_grp[str(db)].attrs["interpretation"] = "image"
 
     @staticmethod
     def program_name():
@@ -637,73 +630,72 @@ class SADeltaBetaTask(
         return "Semi automatic center of rotation / axis calculation"
 
     @staticmethod
-    def process_to_tomwer_processes(scan):
-        if scan.process_file is not None:
-            entry = "entry"
-            if isinstance(scan, NXtomoScan):
-                entry = scan.entry
+    def get_results_url(scan):
+        return DataUrl(
+            file_path=scan.get_relative_file(
+                os.path.join(DEFAULT_RECONS_FOLDER, "tomwer_sadelta_beta.h5")
+            ),
+            data_path=scan.entry or "entry",
+            scheme="silx",
+        )
 
-            db = None
-            if scan.sa_delta_beta_params is not None:
-                db = scan.sa_delta_beta_params.selected_delta_beta_value
+    @staticmethod
+    def save_results_to_disk(scan):
+        if scan.saaxis_params.scores in (None, {}):
+            return
 
-            process_index = scan.pop_process_index()
-            try:
-                with scan.acquire_process_file_lock():
-                    Task._register_process(
-                        process_file=scan.process_file,
-                        entry=entry,
-                        results={"delta_beta": db if db is not None else "-"},
-                        configuration=scan.sa_delta_beta_params.to_dict(),
-                        process_index=process_index,
-                        overwrite=True,
-                        process=SADeltaBetaTask,
-                    )
-                    SADeltaBetaTask._extends_results(
-                        scan=scan, entry=entry, process_index=process_index
-                    )
-            except Exception as e:
-                _logger.warning(
-                    f"Fail to register process of with index {process_index}. Reason is {e}"
+        results_url = SADeltaBetaTask.get_results_url(scan=scan)
+        # save it to the file
+        with HDF5File(results_url.file_path(), mode="w") as h5f:
+            nx_process = h5f.require_group(results_url.data_path())
+            if "NX_class" not in nx_process.attrs:
+                nx_process.attrs["NX_class"] = "NXprocess"
+
+            results = nx_process.require_group("results")
+
+            for cor, (url, score) in scan.sa_delta_beta_params.scores.items():
+                results_db = results.require_group(str(cor))
+                for method in ScoreMethod:
+                    if method is ScoreMethod.TOMO_CONSISTENCY:
+                        continue
+                    results_db[method.value] = score.get(method)
+
+                link_path = os.path.relpath(
+                    url.file_path(),
+                    os.path.dirname(results_url.file_path()),
+                )
+                results_db["reconstructed_slice"] = h5py.ExternalLink(
+                    link_path, url.data_path()
                 )
-
-            return process_index
 
     @staticmethod
-    def _extends_results(scan, entry, process_index):
-        process_file = scan.process_file
-        process_name = "tomwer_process_" + str(process_index)
+    def load_results_from_disk(scan):
+        results_url = SADeltaBetaTask.get_results_url(scan=scan)
+        process_file = results_url.file_path()
 
-        if scan.sa_delta_beta_params.scores in (None, {}):
-            return
-
-        def get_process_path():
-            return "/".join((entry or "entry", process_name))
+        if process_file is None or not os.path.exists(process_file):
+            _logger.warning(
+                "Unable to find process file. Unable to read " "existing processing"
+            )
+            return None, None
 
-        # save it to the file
-        with FileLockerManager().get_lock(process_file):
-            # needs an extra lock for multiprocessing
-
-            with HDF5File(process_file, mode="a") as h5f:
-                nx_process = h5f.require_group(get_process_path())
-                if "NX_class" not in nx_process.attrs:
-                    nx_process.attrs["NX_class"] = "NXprocess"
-
-                results = nx_process.require_group("results")
-                for cor, (url, score) in scan.sa_delta_beta_params.scores.items():
-                    results_db = results.require_group(str(cor))
-                    for method in ScoreMethod:
-                        if method is ScoreMethod.TOMO_CONSISTENCY:
-                            continue
-                        results_db[method.value] = score.get(method)
-
-                    link_path = os.path.relpath(
-                        url.file_path(),
-                        os.path.dirname(process_file),
-                    )
-                    results_db["reconstructed_slice"] = h5py.ExternalLink(
-                        link_path, url.data_path()
+        try:
+            with EntryReader(results_url) as h5f_entry_node:
+                scores = core_utils.get_scores(h5f_entry_node)
+                if (
+                    "results" in h5f_entry_node
+                    and "delta_beta" in h5f_entry_node["results"]
+                ):
+                    selected = h5py_read_dataset(
+                        h5f_entry_node["results"]["delta_beta"]
                     )
+                else:
+                    _logger.warning(f"no results found for {scan}")
+                    selected = None
+            return scores, selected
+        except ValueError:
+            _logger.warning(f"Data path ({results_url.data_path()}) not found")
+            return None, None
 
     def cancel(self):
         """

tomwer/core/process/reconstruction/tests/test_params_cache.py

@@ -0,0 +1,37 @@
+import os
+from tomwer.core.process.reconstruction.params_cache import (
+    _CACHE,
+    _CACHE_SIZE,
+    save_reconstruction_parameters_to_cache,
+    clear_reconstruction_parameters_cache,
+)
+from tomwer.core.utils.scanutils import MockNXtomo
+
+
+def test_cache(tmp_path):
+    nxtomos = [
+        MockNXtomo(
+            scan_path=os.path.join(tmp_path, f"my_nxtomo_{i_scan}"),
+            n_proj=10,
+        ).scan
+        for i_scan in range(31)
+    ]
+    new_nxtomo = MockNXtomo(
+        scan_path=os.path.join(tmp_path, "my_nxtomo_99"),
+        n_proj=10,
+    ).scan
+
+    clear_reconstruction_parameters_cache()
+    # test saving it in the original order
+    [save_reconstruction_parameters_to_cache(scan=scan) for scan in nxtomos]
+    assert len(_CACHE) == _CACHE_SIZE
+    assert _CACHE == {
+        nxtomo.get_identifier().to_str(): (None, None) for nxtomo in nxtomos[1:]
+    }
+
+    # try updating the next last one to make sure it won't be removed
+    save_reconstruction_parameters_to_cache(scan=nxtomos[1])
+    save_reconstruction_parameters_to_cache(scan=new_nxtomo)
+    assert nxtomos[1].get_identifier().to_str() in _CACHE
+    assert nxtomos[2].get_identifier().to_str() not in _CACHE
+    clear_reconstruction_parameters_cache()

tomwer/core/process/script/python.py

@@ -7,7 +7,6 @@ import logging
 
 import tomwer.version
 from tomwer.core.process.task import Task
-from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.scanutils import data_identifier_to_scan
@@ -40,24 +39,6 @@ class PythonScript(
         out_data = data_identifier_to_scan(interpreter.locals.get("out_data"))
         out_volume = data_identifier_to_scan(interpreter.locals.get("out_volume"))
 
-        # register process
-        if isinstance(scan, EDFTomoScan):
-            entry = None
-        else:
-            entry = scan.entry
-
-        try:
-            self.register_process(
-                process_file=scan.process_file,
-                entry=entry,
-                configuration={"scriptText": self.get_configuration()["scriptText"]},
-                results=None,
-                process_index=scan.pop_process_index(),
-                overwrite=True,
-            )
-        except Exception as e:
-            _logger.error("Fail to register process. Error is " + str(e))
-
         if out_data is not None and self.get_input_value("serialize_output_data", True):
             self.outputs.data = out_data.to_dict()
         else: