tomwer 1.2.8__py3-none-any.whl → 1.3.0a0__py3-none-any.whl

tomwer/synctools/stacks/reconstruction/nabu.py

@@ -36,7 +36,7 @@ from silx.gui.utils import blockSignals
 
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.settings import get_lbsram_path, isOnLbsram
-from tomwer.core.utils import isLowOnMemory
+from tomwer.core.utils.lbsram import is_low_on_memory
 from tomwer.core.process.reconstruction.nabu.nabuslices import NabuSlicesTask
 from tomwer.core.process.reconstruction.nabu.nabuvolume import NabuVolumeTask
 
@@ -66,7 +66,7 @@ class NabuSliceProcessStack(FIFO, qt.QObject):
         assert isinstance(data, TomwerScanBase)
         self._computationThread.finished.connect(self._end_threaded_computation)
 
-        if isOnLbsram(data) and isLowOnMemory(get_lbsram_path()) is True:
+        if isOnLbsram(data) and is_low_on_memory(get_lbsram_path()) is True:
             # if computer is running into low memory on lbsram skip it
             mess = f"low memory, skip nabu reconstruction for {data.path}"
             _logger.processSkipped(mess)

tomwer/synctools/stacks/reconstruction/normalization.py

@@ -37,7 +37,7 @@ from silx.gui import qt
 from tomwer.core.process.reconstruction.normalization import SinoNormalizationTask
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.settings import get_lbsram_path, isOnLbsram
-from tomwer.core.utils import isLowOnMemory
+from tomwer.core.utils.lbsram import is_low_on_memory
 
 from ..processingstack import FIFO, ProcessingThread
 
@@ -60,7 +60,7 @@ class INormalizationProcessStack(FIFO, qt.QObject):
         )
         _logger.processStarted(f"start intensity normalization {data}")
         assert isinstance(data, TomwerScanBase)
-        if isOnLbsram(data) and isLowOnMemory(get_lbsram_path()) is True:
+        if isOnLbsram(data) and is_low_on_memory(get_lbsram_path()) is True:
             # if computer is running into low memory on lbsram skip it
             mess = "low memory, skip intensity normalization", data.path
             ProcessManager().notify_dataset_state(

tomwer/synctools/stacks/reconstruction/saaxis.py

@@ -37,7 +37,7 @@ from silx.gui import qt
 from tomwer.core.process.reconstruction.saaxis import SAAxisTask
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.settings import get_lbsram_path, isOnLbsram
-from tomwer.core.utils import isLowOnMemory
+from tomwer.core.utils.lbsram import is_low_on_memory
 from tomwer.synctools.axis import QAxisRP
 from tomwer.synctools.saaxis import QSAAxisParams
 
@@ -78,7 +78,7 @@ class SAAxisProcessStack(FIFO, qt.QObject):
         self._data_currently_computed = data
         saaxis_params = QSAAxisParams.from_dict(configuration)
         saaxis_params.frame_width = data.dim_1
-        if isOnLbsram(data) and isLowOnMemory(get_lbsram_path()) is True:
+        if isOnLbsram(data) and is_low_on_memory(get_lbsram_path()) is True:
             # if computer is running into low memory on lbsram skip it
             mess = f"low memory, skip saaxis calculation {data.path}"
             ProcessManager().notify_dataset_state(

tomwer/synctools/stacks/reconstruction/sadeltabeta.py

@@ -39,7 +39,7 @@ from tomwer.core.process.reconstruction.sadeltabeta.sadeltabeta import (
 )
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.settings import get_lbsram_path, isOnLbsram
-from tomwer.core.utils import isLowOnMemory
+from tomwer.core.utils.lbsram import is_low_on_memory
 from tomwer.synctools.axis import QAxisRP
 from tomwer.synctools.sadeltabeta import QSADeltaBetaParams
 
@@ -79,7 +79,7 @@ class SADeltaBetaProcessStack(FIFO, qt.QObject):
             data.sa_delta_beta_params = QSADeltaBetaParams()
         self._data_currently_computed = data
         sa_delta_beta_params = QSADeltaBetaParams.from_dict(configuration)
-        if isOnLbsram(data) and isLowOnMemory(get_lbsram_path()) is True:
+        if isOnLbsram(data) and is_low_on_memory(get_lbsram_path()) is True:
             # if computer is running into low memory on lbsram skip it
             mess = f"low memory, skip sa-delta-beta-axis calculation {data.path}"
             ProcessManager().notify_dataset_state(

tomwer/synctools/test/test_darkRefs.py

@@ -38,9 +38,8 @@ from silx.gui.utils.testutils import TestCaseQt
 
 from tomwer.core.process.reconstruction.darkref.darkrefs import DarkRefsTask
 from tomwer.core.process.reconstruction.darkref.params import DKRFRP, ReduceMethod
-from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.scanutils import MockEDF
-from tomwer.tests.utils import UtilsTest
+from tomwer.tests.datasets import TomwerCIDatasets
 
 
 class TestDarkRefsBehavior(TestCaseQt):
@@ -58,7 +57,9 @@ class TestDarkRefsBehavior(TestCaseQt):
         """number of theoretical dark file the algorithm should create"""
         self.dataset_folder = os.path.join(self.tmpDir, self.datasetsID)
 
-        dataDir = UtilsTest.getEDFDataset(self.datasetsID)
+        dataDir = TomwerCIDatasets.get_dataset(
+            f"edf_datasets/{self.datasetsID}",
+        )
         shutil.copytree(dataDir, self.dataset_folder)
         files = os.listdir(self.dataset_folder)
         for _f in files:
@@ -380,7 +381,9 @@ class TestDarkAccumulation(TestCaseQt):
     def setUp(self):
         super().setUp()
         self.dataset = "bone8_1_"
-        dataDir = UtilsTest.getEDFDataset(self.dataset)
+        dataDir = TomwerCIDatasets.get_dataset(
+            f"edf_datasets/{self.dataset}",
+        )
         self.outputdir = tempfile.mkdtemp()
         shutil.copytree(src=dataDir, dst=os.path.join(self.outputdir, self.dataset))
         self.darkFile = os.path.join(self.outputdir, self.dataset, "dark.edf")
@@ -418,57 +421,3 @@ class TestDarkAccumulation(TestCaseQt):
         with fabio.open(self.darkFile) as dsc:
             self.computed_dark = dsc.data
         self.assertTrue(numpy.array_equal(self.computed_dark, self.dark_reference))
-
-
-class TestPCOTomo(TestCaseQt):
-    """Test processing of DKRF are correct"""
-
-    def setUp(self):
-        TestCaseQt.setUp(self)
-        self.tmp_dir = tempfile.mkdtemp()
-        MockEDF.fastMockAcquisition(self.tmp_dir)
-
-        self.recons_params = DKRFRP()
-        self.recons_params.flat_calc_method = ReduceMethod.NONE
-        self.recons_params.dark_calc_method = ReduceMethod.NONE
-        self.recons_params.dark_pattern = ".*_dark_.*"
-        self.recons_params.flat_pattern = ".*_ref_.*"
-        self.recons_params._set_remove_opt(True)
-
-    def copyDataset(self, dataset):
-        folder = os.path.join(self.tmp_dir, dataset)
-        shutil.copytree(os.path.join(UtilsTest.getEDFDataset(dataset)), folder)
-        self.scan = ScanFactory.create_scan_object(scan_path=folder)
-
-    def tearDown(self):
-        shutil.rmtree(self.tmp_dir)
-
-    def testDark2x2Scan(self):
-        self.dataset = "pcotomo_2x2scan_refdarkbeg_end_conti"
-        self.copyDataset(self.dataset)
-        self.recons_params.dark_calc_method = ReduceMethod.MEDIAN
-        dar_ref_process = DarkRefsTask(
-            inputs={
-                "dark_ref_params": self.recons_params,
-                "data": self.scan,
-                "darkhst_prefix": "darkHST",
-                "force_sync": True,
-                "serialize_output_data": False,
-            }
-        )
-        dar_ref_process.run()
-
-    def testRef2x2Scan(self):
-        self.dataset = "pcotomo_2x2scan_refdarkbeg_end_conti"
-        self.copyDataset(self.dataset)
-        self.recons_params.flat_calc_method = ReduceMethod.MEDIAN
-        dar_ref_process = DarkRefsTask(
-            inputs={
-                "dark_ref_params": self.recons_params,
-                "data": self.scan,
-                "darkhst_prefix": "darkHST",
-                "force_sync": True,
-                "serialize_output_data": False,
-            }
-        )
-        dar_ref_process.run()

tomwer/synctools/test/test_foldertransfer.py

@@ -40,12 +40,12 @@ from nxtomomill.converter import from_h5_to_nx
 from nxtomomill.io.config.hdf5config import TomoHDF5Config
 from silx.gui.utils.testutils import TestCaseQt
 
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanfactory import ScanFactory
-from tomwer.core.utils import rebaseParFile
+from tomwer.core.utils.spec import rebaseParFile
 from tomwer.core.utils.scanutils import MockEDF
 from tomwer.synctools.datatransfert import ScanTransfer
-from tomwer.tests.utils import UtilsTest
+from tomwer.tests.datasets import TomwerCIDatasets
 
 try:
     from tomwer.synctools.rsyncmanager import RSyncManager
@@ -239,7 +239,7 @@ class TestHDFDataTransfert(TestCaseQt):
         self.input_dir = tempfile.mkdtemp()
         self.output_dir = tempfile.mkdtemp()
         shutil.copytree(
-            UtilsTest.getBlissDataset(folderID="sample"),
+            TomwerCIDatasets.get_dataset("bliss/sample"),
             os.path.join(self.input_dir, "sample"),
         )
 
@@ -272,7 +272,7 @@ class TestHDFDataTransfert(TestCaseQt):
         output_file_path, entry = results[0]
         assert os.path.exists(output_file_path)
         assert os.path.isfile(output_file_path)
-        self.scan = HDF5TomoScan(scan=output_file_path, entry=entry)
+        self.scan = NXtomoScan(scan=output_file_path, entry=entry)
 
         # register information regarding origin of the file
         from tomwer.core.process.control.datalistener import DataListener
@@ -333,7 +333,7 @@ class TestPreTransfert(unittest.TestCase):
 
     def setUp(self):
         unittest.TestCase.setUp(self)
-        folderDataset = UtilsTest.getEDFDataset("scan_3_")
+        folderDataset = TomwerCIDatasets.get_dataset("edf_datasets/scan_3_")
         self.tmpdir = tempfile.mkdtemp()
         self.outputfolder = tempfile.mkdtemp()
         scan_path = os.path.join(self.tmpdir, "scan_3_")

tomwer/synctools/utils/scanstages.py

@@ -38,7 +38,7 @@ from silx.io.url import DataUrl
 from silx.utils.enum import Enum as _Enum
 
 from tomwer.core.scan.edfscan import EDFTomoScan
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.nxtomoscan import NXtomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.synctools.rsyncmanager import RSyncManager
 
@@ -117,7 +117,7 @@ class ScanStages:
                     RSyncManager().sync_file(
                         source=info_file, target=file_info_dest, wait=True
                     )
-            elif isinstance(self.scan, HDF5TomoScan):
+            elif isinstance(self.scan, NXtomoScan):
                 raise NotImplementedError()
         elif stage is ScanStages.AcquisitionStage.ACQUI_ON_GOING:
             # copy half of the acquisition file
@@ -129,7 +129,7 @@ class ScanStages:
                     n_url -= 1
                     self._copy_url_file(url=urls[keys[n_url]], dest_dir=dest_dir)
 
-            elif isinstance(self.scan, HDF5TomoScan):
+            elif isinstance(self.scan, NXtomoScan):
                 raise NotImplementedError()
         elif stage is ScanStages.AcquisitionStage.ACQUI_ENDED:
             if isinstance(self.scan, EDFTomoScan):
@@ -142,7 +142,7 @@ class ScanStages:
                 )
                 xml_dest = os.path.join(dest_dir, os.path.basename(xml_file))
                 RSyncManager().sync_file(source=xml_file, target=xml_dest, wait=True)
-            elif isinstance(self.scan, HDF5TomoScan):
+            elif isinstance(self.scan, NXtomoScan):
                 raise NotImplementedError()
         elif stage is ScanStages.AcquisitionStage.RECONSTRUCTION_ADDED:
             # copy reconstruction
@@ -188,14 +188,14 @@ class ScanStages:
                     RSyncManager().sync_file(
                         source=_xml_file, target=xml_dest, wait=True
                     )
-            elif isinstance(self.scan, HDF5TomoScan):
+            elif isinstance(self.scan, NXtomoScan):
                 raise NotImplementedError()
         elif stage is ScanStages.AcquisitionStage.COMPLETE:
             if isinstance(self.scan, EDFTomoScan):
                 for file_ in os.listdir(self.scan.path):
                     file_fp = os.path.join(self.scan.path, file_)
                     RSyncManager().sync_file(source=file_fp, target=dest_dir, wait=True)
-            elif isinstance(self.scan, HDF5TomoScan):
+            elif isinstance(self.scan, NXtomoScan):
                 raise NotImplementedError()
         else:
             raise ValueError("given stage is not recognized", stage)

tomwer/tests/conftest.py

@@ -1,4 +1,5 @@
 import gc
+import os
 import logging
 
 import pytest
@@ -49,3 +50,36 @@ def warn_qtwidgets_alive():
             len(widgets),
             "\n ".join(map(str, widgets)),
         )
+
+
+@pytest.fixture(scope="function")
+def nxtomo_scan_180(tmp_path):
+    test_path = tmp_path / "test"
+    test_path.mkdir()
+    yield create_default_scan(
+        scan_path=os.path.join(test_path, "scan_180"),
+        scan_range=180,
+    )
+
+
+@pytest.fixture(scope="function")
+def nxtomo_scan_360(tmp_path):
+    test_path = tmp_path / "test"
+    test_path.mkdir()
+    yield create_default_scan(
+        scan_path=os.path.join(test_path, "scan_360"),
+        scan_range=360,
+    )
+
+
+def create_default_scan(scan_path: str, scan_range: int):
+    from tomwer.core.utils.scanutils import MockNXtomo
+
+    return MockNXtomo(
+        scan_path=scan_path,
+        n_proj=10,
+        n_ini_proj=10,
+        scan_range=scan_range,
+        dim=20,
+        energy=12.3,
+    ).scan

tomwer/tests/datasets.py

@@ -0,0 +1,13 @@
+import os
+from tomoscan.test.datasets import GitlabProject
+
+TomwerCIDatasets = GitlabProject(
+    branch_name="tomwer",
+    host="https://gitlab.esrf.fr",
+    cache_dir=os.path.join(
+        os.path.dirname(__file__),
+        "__archive__",
+    ),
+    token=None,
+    project_id=4299,  # https://gitlab.esrf.fr/tomotools/ci_datasets
+)

tomwer/tests/test_scripts.py

@@ -1,44 +1,10 @@
-# coding: utf-8
-# /*##########################################################################
-# Copyright (C) 2016 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-#############################################################################*/
-
-
-from __future__ import absolute_import, division, print_function
-
 """Test suite to scripts"""
 
-__authors__ = ["V.Valls", "H.Payno"]
-__license__ = "MIT"
-__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "21/02/2018"
-
-
 import logging
 import subprocess
 import sys
 import unittest
-
-from tomwer.tests.utils.utilstest import UtilsTest
+import os
 
 _logger = logging.getLogger(__name__)
 
@@ -77,8 +43,9 @@ class TestScriptsHelp(unittest.TestCase):
         self.assertEqual(p.returncode, 0)
 
     def executeAppHelp(self, script_name, module_name):
-        script = UtilsTest.script_path(script_name, module_name)
-        env = UtilsTest.get_test_env()
+        test_script = ScriptTest()
+        script = test_script.script_path(script_name, module_name)
+        env = test_script.get_test_env()
         if script.endswith(".exe"):
             command_line = [script]
         else:
@@ -89,23 +56,109 @@ class TestScriptsHelp(unittest.TestCase):
     def testAxis(self):
         self.executeAppHelp("axis", "tomwer.app.axis")
 
+    def testCanvas(self):
+        self.executeAppHelp("axis", "tomwer.app.canvas")
+
     def testDarkRef(self):
         self.executeAppHelp("darkref", "tomwer.app.darkref")
 
+    def testDarkRefPath(self):
+        self.executeAppHelp("darkrefpatch", "tomwer.app.darkrefpatch")
+
+    def testDiffFrame(self):
+        self.executeAppHelp("diffFrame", "tomwer.app.diffframe")
+
+    def testImageKeyEditor(self):
+        self.executeAppHelp("image-key-editor", "tomwer.app.imagekeyeditor")
+
+    def testImageKeyUpgrader(self):
+        self.executeAppHelp("image-key-upgrader", "tomwer.app.imagekeyupgrader")
+
+    def testIntensityNormalization(self):
+        self.executeAppHelp(
+            "intensity-normalization", "tomwer.app.intensitynormalization"
+        )
+
     def testLamino(self):
         self.executeAppHelp("lamino", "tomwer.app.lamino")
 
+    def testMulticor(self):
+        self.executeAppHelp("multicor", "tomwer.app.multicor")
+
+    def testMultiPag(self):
+        self.executeAppHelp("multipag", "tomwer.app.multipag")
+
     def testNabu(self):
         self.executeAppHelp("nabu", "tomwer.app.nabuapp")
 
+    def testNxTomoEditor(self):
+        self.executeAppHelp("nxtomo-editor", "tomwer.app.nxtomoeditor")
+
+    def testPatchRawDarkflat(self):
+        self.executeAppHelp("patch-raw-dark-flat", "tomwer.app.patchrawdarkflat")
+
     def testRadioStack(self):
         self.executeAppHelp("radiostack", "tomwer.app.radiostack")
 
+    def testReduceDarkFlat(self):
+        self.executeAppHelp("reduce-dark-flat", "tomwer.app.reducedarkflat")
+
+    def testRSync(self):
+        self.executeAppHelp("rsync", "tomwer.app.rsync")
+
     def testSampleMoved(self):
         self.executeAppHelp("samplemoved", "tomwer.app.samplemoved")
 
+    def testScanviewer(self):
+        self.executeAppHelp("scanviewer", "tomwer.app.scanviewer")
+
+    def testSinogramViewer(self):
+        self.executeAppHelp("sinogramviewer", "tomwer.app.sinogramviewer")
+
     def testSliceStack(self):
         self.executeAppHelp("slicestack", "tomwer.app.slicestack")
 
-    def testRSync(self):
-        self.executeAppHelp("rsync", "tomwer.app.rsync")
+    def testStopDataListener(self):
+        self.executeAppHelp("stop-data-listener", "tomwer.app.stopdatalistener")
+
+    def testZStitching(self):
+        self.executeAppHelp("zstitching", "tomwer.app.zstitching")
+
+
+class ScriptTest:
+    """
+    Class providing useful stuff for preparing script tests.
+    """
+
+    def get_test_env(self):
+        """
+        Returns an associated environment with a working project.
+        """
+        env = dict((str(k), str(v)) for k, v in os.environ.items())
+        env["PYTHONPATH"] = os.pathsep.join(sys.path)
+        return env
+
+    def script_path(self, script_name, module_name):
+        """Returns the script path according to it's location"""
+        import importlib
+
+        module = importlib.import_module(module_name)
+        script = module.__file__
+        return script
+
+    def get_installed_script_path(self, script):
+        """
+        Returns the path of the executable and the associated environment
+
+        In Windows, it checks availability of script using .py .bat, and .exe
+        file extensions.
+        """
+        paths = os.environ.get("PATH", "").split(os.pathsep)
+        for base in paths:
+            # clean up extra quotes from paths
+            if base.startswith('"') and base.endswith('"'):
+                base = base[1:-1]
+        # script not found
+        _logger.warning("Script '%s' not found in paths: %s", script, ":".join(paths))
+        script_path = script
+        return script_path

tomwer/tests/utils.py

@@ -0,0 +1,5 @@
+import os
+
+
+def skip_gui_test():
+    return os.environ.get("_TOMWER_NO_GUI_UNIT_TESTS", "False") == "True"

tomwer/version.py

@@ -76,9 +76,9 @@ RELEASE_LEVEL_VALUE = {
 }
 
 MAJOR = 1
-MINOR = 2
-MICRO = 8
-RELEV = "final"  # <16
+MINOR = 3
+MICRO = 0
+RELEV = "alpha"  # <16
 SERIAL = 0  # <16
 
 date = __date__