tomwer 1.1.0rc7__py3-none-any.whl → 1.2.0a0__py3-none-any.whl

tomwer/core/process/reconstruction/nabu/nabucommon.py

@@ -32,37 +32,36 @@ __license__ = "MIT"
 __date__ = "27/10/2021"
 
 
-from tomwer.core.cluster.cluster import (
-    SlurmClusterConfiguration,
-)
+import logging
+import os
+import subprocess
+import uuid
+from typing import Iterable, Optional, Union
+
+import numpy
+from silx.io.url import DataUrl
+from silx.utils.enum import Enum as _Enum
 from sluurp.executor import submit as submit_to_slurm_cluster
 from sluurp.job import SBatchScriptJob
+from tomoscan.io import HDF5File
+from tomoscan.normalization import Method as INormMethod
+
+from tomwer.core.cluster.cluster import SlurmClusterConfiguration
 from tomwer.core.process.reconstruction.nabu.target import Target
 from tomwer.core.process.reconstruction.nabu.utils import (
     _NabuPhaseMethod,
     nabu_std_err_has_error,
 )
-from tomwer.core.utils.scanutils import format_output_location
+from tomwer.core.process.reconstruction.normalization.params import (
+    _ValueSource as INormSource,
+)
 from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.hdf5scan import HDF5TomoScan
 from tomwer.core.scan.scanbase import TomwerScanBase
+from tomwer.core.utils.scanutils import format_output_location
 from tomwer.core.utils.slurm import get_slurm_script_name, is_slurm_available
-from tomwer.core.process.reconstruction.normalization.params import (
-    _ValueSource as INormSource,
-)
-from tomoscan.normalization import Method as INormMethod
-from silx.utils.enum import Enum as _Enum
-from silx.io.url import DataUrl
-from typing import Iterable, Optional
-from typing import Union
-import logging
-import subprocess
-from . import settings
-from . import utils
-import os
-import uuid
-from tomoscan.io import HDF5File
-import numpy
+
+from . import settings, utils
 
 _logger = logging.getLogger(__name__)
 try:

tomwer/core/process/reconstruction/nabu/nabuscores.py

@@ -47,29 +47,31 @@ except (ImportError, OSError):
         has_nabu = True
 else:
     has_nabu = True
+import logging
+import os
 from typing import Iterable, Optional, Union
 
+from nabu.pipeline.fullfield.nabu_config import (
+    nabu_config as nabu_fullfield_default_config,
+)
 from processview.core.manager.manager import ProcessManager
+
+from tomwer.core.futureobject import FutureTomwerObject
+from tomwer.core.process.reconstruction.nabu.nabuslices import (
+    SingleSliceRunner,
+    _NabuBaseReconstructor,
+    generate_nabu_configfile,
+)
 from tomwer.core.process.reconstruction.nabu.target import Target
 from tomwer.core.progress import Progress
 from tomwer.core.scan.edfscan import EDFTomoScan
-from tomwer.core.futureobject import FutureTomwerObject
 from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.utils.slurm import is_slurm_available
 from tomwer.utils import docstring
+
 from ..nabu import settings as nabu_settings
-from tomwer.core.process.reconstruction.nabu.nabuslices import (
-    SingleSliceRunner,
-    generate_nabu_configfile,
-    _NabuBaseReconstructor,
-)
-from nabu.pipeline.fullfield.nabu_config import (
-    nabu_config as nabu_fullfield_default_config,
-)
-from .nabucommon import ResultSlurmRun, ResultsLocalRun, ResultsWithStd
 from . import utils
-import logging
-import os
+from .nabucommon import ResultsLocalRun, ResultSlurmRun, ResultsWithStd
 
 _logger = logging.getLogger(__name__)
 
@@ -254,7 +256,7 @@ class _Reconstructor(_NabuBaseReconstructor):
                 db=db,
                 ctf=ctf,
             )
-            _config["output"]["file_prefix"] = "cor_{}_{}".format(_file_name, cor)
+            _config["output"]["file_prefix"] = f"cor_{_file_name}_{cor}"
             if _config["output"]["location"] not in ("", None):
                 # if user specify the location
                 if not os.path.isdir(_config["output"]["location"]):
@@ -303,5 +305,5 @@ class _Reconstructor(_NabuBaseReconstructor):
         )
         if not isinstance(self.scan, EDFTomoScan):
             name = "_".join((self.scan.entry.lstrip("/"), name))
-        conf_file = os.path.join(cfg_folder, "cor_{}_{}".format(cor, name))
+        conf_file = os.path.join(cfg_folder, f"cor_{cor}_{name}")
         return config, conf_file

tomwer/core/process/reconstruction/nabu/nabuslices.py

@@ -28,15 +28,16 @@ __license__ = "MIT"
 __date__ = "26/10/2021"
 
 
-import functools
-import os
 import copy
+import functools
 import logging
+import os
+
 from processview.core.manager.manager import ProcessManager
 
 from tomwer.core.futureobject import FutureTomwerObject
-from tomwer.core.utils.slurm import is_slurm_available
 from tomwer.core.utils.scanutils import data_identifier_to_scan
+from tomwer.core.utils.slurm import is_slurm_available
 
 try:
     from nabu.pipeline.fullfield.reconstruction import (  # noqa F401
@@ -58,36 +59,40 @@ else:
     has_nabu = True
     _err_import_nabu = None
 
-from silx.utils.enum import Enum as _Enum
-from tomwer.core.scan.scanbase import TomwerScanBase
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
-from tomwer.core.scan.scanfactory import ScanFactory
-from tomwer.utils import docstring
-from typing import Iterable, Union
-from typing import Optional
-from nabu.pipeline.config import generate_nabu_configfile
+from typing import Iterable, Optional, Union
+
+from nabu import version as nabu_version
+from nabu.pipeline.config import (
+    _extract_nabuconfig_keyvals,
+    generate_nabu_configfile,
+    get_default_nabu_config,
+)
 from nabu.pipeline.fullfield.nabu_config import (
     nabu_config as nabu_fullfield_default_config,
 )
-from nabu.pipeline.config import _extract_nabuconfig_keyvals
-from nabu import version as nabu_version
+from processview.core.superviseprocess import SuperviseProcess
 from silx.io.dictdump import h5todict
-from . import utils
-from . import settings as nabu_settings
+from silx.io.utils import h5py_read_dataset
+from silx.utils.enum import Enum as _Enum
 from tomoscan.io import HDF5File
+
+from tomwer.core.process.task import Task
 from tomwer.core.scan.edfscan import EDFTomoScan
-from silx.io.utils import h5py_read_dataset
-from nabu.pipeline.config import get_default_nabu_config
+from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.scanbase import TomwerScanBase
+from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.io.utils.h5pyutils import EntryReader
-from tomwer.core.process.task import Task
-from .target import Target
+from tomwer.utils import docstring
+
+from . import settings as nabu_settings
+from . import utils
 from .nabucommon import (
-    _NabuBaseReconstructor,
-    ResultsRun,
     ResultsLocalRun,
     ResultSlurmRun,
+    ResultsRun,
+    _NabuBaseReconstructor,
 )
-from processview.core.superviseprocess import SuperviseProcess
+from .target import Target
 
 _logger = logging.getLogger(__name__)
 if not has_nabu:

tomwer/core/process/reconstruction/nabu/nabuvolume.py

@@ -28,36 +28,34 @@ __license__ = "MIT"
 __date__ = "26/10/2021"
 
 
+import copy
+import functools
 import logging
+import os
 from typing import Iterable, Optional
+
+from nabu import version as nabu_version
+from nabu.pipeline.config import generate_nabu_configfile
+from nabu.pipeline.fullfield.nabu_config import (
+    nabu_config as nabu_fullfield_default_config,
+)
 from processview.core.manager.manager import ProcessManager
+from silx.io.utils import h5py_read_dataset
+
 from tomwer.core.cluster.cluster import SlurmClusterConfiguration
 from tomwer.core.futureobject import FutureTomwerObject
-from tomwer.core.scan.scanbase import TomwerScanBase
+from tomwer.core.process.task import Task
+from tomwer.core.scan.edfscan import EDFTomoScan
 from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.scanbase import TomwerScanBase
 from tomwer.core.scan.scanfactory import ScanFactory
 from tomwer.core.utils.scanutils import data_identifier_to_scan
-from . import utils
-from . import settings
-from nabu.pipeline.config import generate_nabu_configfile
-from nabu.pipeline.fullfield.nabu_config import (
-    nabu_config as nabu_fullfield_default_config,
-)
-from nabu import version as nabu_version
-from tomwer.core.scan.edfscan import EDFTomoScan
-import copy
-import os
-from tomwer.core.process.task import Task
 from tomwer.io.utils.h5pyutils import EntryReader
-from silx.io.utils import h5py_read_dataset
-from .target import Target
-import functools
 from tomwer.utils import docstring
-from .nabucommon import (
-    _NabuBaseReconstructor,
-    ResultsLocalRun,
-    ResultSlurmRun,
-)
+
+from . import settings, utils
+from .nabucommon import ResultsLocalRun, ResultSlurmRun, _NabuBaseReconstructor
+from .target import Target
 
 _logger = logging.getLogger(__name__)
 
@@ -260,7 +258,7 @@ class VolumeRunner(_NabuBaseReconstructor):
         if not isinstance(self.scan, EDFTomoScan):
             name = "_".join((self.scan.entry.lstrip("/"), name))
         conf_file = os.path.join(cfg_folder, name)
-        _logger.info("{}: create {}".format(self.scan, conf_file))
+        _logger.info(f"{self.scan}: create {conf_file}")
 
         # add some tomwer metadata and save the configuration
         # note: for now the section is ignored by nabu but shouldn't stay that way
@@ -401,7 +399,7 @@ class NabuVolume(
         elif isinstance(scan, dict):
             scan = ScanFactory.create_scan_object_frm_dict(scan)
         else:
-            raise ValueError("input type {} is not managed".format(scan))
+            raise ValueError(f"input type {scan} is not managed")
 
         if scan.nabu_recons_params is None:
             raise ValueError(

tomwer/core/process/reconstruction/nabu/slurm.py

@@ -0,0 +1,36 @@
+import subprocess
+import logging
+
+_logger = logging.getLogger(__name__)
+
+
+def check_n_gpu():
+    try:
+        from pycuda import driver  # pylint: disable=E0401
+
+        driver.init()
+        n_gpus = driver.Device.count()
+    except Exception as e:
+        raise e
+    else:
+        if n_gpus == 0:
+            raise RuntimeError("No gpu detected to run nabu reconstruction")
+        else:
+            _logger.info(f"{n_gpus} detected by pycuda")
+
+
+def _exec_nabu_on_slurm(conf_file: str, scan_path: str) -> tuple:
+    check_n_gpu()
+
+    NABU_FULL_FIELD_APP_PATH = "nabu.app.reconstruct"
+    # need to be executed in his own context
+    command = " ".join(("python", "-m", NABU_FULL_FIELD_APP_PATH, conf_file))
+    process = subprocess.Popen(
+        command,
+        shell=True,
+        cwd=scan_path,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    res = process.communicate()
+    return res

tomwer/core/process/reconstruction/nabu/utils.py

@@ -28,25 +28,27 @@ __license__ = "MIT"
 __date__ = "06/08/2020"
 
 
-from tomwer.core.scan.edfscan import EDFTomoScan
-from tomwer.core.scan.scanbase import TomwerScanBase
-from tomwer.core.scan.hdf5scan import HDF5TomoScan
-import tomwer.version
-from silx.utils.enum import Enum as _Enum
-from contextlib import AbstractContextManager
-import logging
 import datetime
-import typing
+import logging
 import os
-from tomwer.core.volume.hdf5volume import HDF5Volume
-from tomwer.core.volume.edfvolume import EDFVolume
-from tomwer.core.volume.jp2kvolume import JP2KVolume
-from tomwer.core.volume.tiffvolume import TIFFVolume
-from tomwer.core.volume.rawvolume import RawVolume
+import typing
+from contextlib import AbstractContextManager
+
+from nabu.pipeline.config import generate_nabu_configfile, parse_nabu_config_file
 from nabu.pipeline.fullfield.nabu_config import (
     nabu_config as nabu_fullfield_default_config,
 )
-from nabu.pipeline.config import generate_nabu_configfile, parse_nabu_config_file
+from silx.utils.enum import Enum as _Enum
+
+import tomwer.version
+from tomwer.core.scan.edfscan import EDFTomoScan
+from tomwer.core.scan.hdf5scan import HDF5TomoScan
+from tomwer.core.scan.scanbase import TomwerScanBase
+from tomwer.core.volume.edfvolume import EDFVolume
+from tomwer.core.volume.hdf5volume import HDF5Volume
+from tomwer.core.volume.jp2kvolume import JP2KVolume
+from tomwer.core.volume.rawvolume import RawVolume
+from tomwer.core.volume.tiffvolume import TIFFVolume
 
 _logger = logging.getLogger(__name__)
 
@@ -242,7 +244,7 @@ def get_recons_volume_identifier(
     #     )
 
     else:
-        raise ValueError("file format not managed: {}".format(file_format))
+        raise ValueError(f"file format not managed: {file_format}")
 
     return tuple([volume.get_identifier() for volume in volumes])
 
@@ -413,6 +415,7 @@ def nabu_std_err_has_error(errs: typing.Optional[bytes]):
         return (
             "warning" in line
             or "Warning" in line
+            or "deprecated" in line
             or line.replace(" ", "") == ""
             or "unable to load" in line
         )

tomwer/core/process/reconstruction/normalization/__init__.py

@@ -1,4 +1,2 @@
-from .normalization import (  # noqa F403
-    SinoNormalizationTask,
-)
+from .normalization import SinoNormalizationTask  # noqa F403
 from .params import SinoNormalizationParams  # noqa F403

tomwer/core/process/reconstruction/normalization/normalization.py

@@ -34,24 +34,22 @@ __license__ = "MIT"
 __date__ = "25/06/2021"
 
 
-import numpy
-import logging
 import functools
+import logging
+
+import numpy
+import tomoscan.esrf.scan.utils
+from processview.core.dataset import DatasetIdentifier
+from processview.core.superviseprocess import SuperviseProcess
 from silx.io.url import DataUrl
+from tomoscan.esrf.scan.utils import get_data
+from tomoscan.normalization import Method as NormMethod
+
 import tomwer.version
 from tomwer.core.process.task import Task
 from tomwer.core.utils.scanutils import data_identifier_to_scan
-from .params import (
-    SinoNormalizationParams,
-    _ValueCalculationFct,
-    _ValueSource,
-)
-from tomoscan.normalization import Method as NormMethod
-import tomoscan.esrf.scan.utils
-from tomoscan.esrf.scan.utils import get_data
-from processview.core.dataset import DatasetIdentifier
-from processview.core.superviseprocess import SuperviseProcess
 
+from .params import SinoNormalizationParams, _ValueCalculationFct, _ValueSource
 
 _logger = logging.getLogger(__name__)
 
@@ -143,7 +141,7 @@ class SinoNormalizationTask(
                     "source": _ValueSource.MANUAL_SCALAR.value,
                 }
             else:
-                raise ValueError("method {} is not handled".format(params.method))
+                raise ValueError(f"method {params.method} is not handled")
         except Exception as e:
             _logger.error(e)
             final_norm_info = {"tomwer_processing_res_code": False}
@@ -274,7 +272,7 @@ class SinoNormalizationTask(
 
             def retrieve_data_proj_indexes(url_):
                 urls = []
-                for slice in range(
+                for data_slice in range(
                     url_.data_slice().start,
                     url_.data_slice().stop,
                     url_.data_slice().step or 1,
@@ -284,7 +282,7 @@ class SinoNormalizationTask(
                             file_path=url_.file_path(),
                             data_path=url_.data_path(),
                             scheme=url_.scheme(),
-                            data_slice=slice,
+                            data_slice=data_slice,
                         )
                     )
 
@@ -298,8 +296,7 @@ class SinoNormalizationTask(
                         res.append(url_idxs[my_url_path])
                     else:
                         _logger.warning(
-                            "unable to retrieve frame index from url {}. "
-                            "Take the slice index as frame index".format(my_url_path)
+                            f"unable to retrieve frame index from url {my_url_path}. Take the slice index as frame index"
                         )
                 return res
 
@@ -323,9 +320,7 @@ class SinoNormalizationTask(
                 ]
                 current_idx += length
             else:
-                raise ValueError(
-                    "Frame where expected and not a " "{}D object".format(data.ndim)
-                )
+                raise ValueError(f"Frame where expected and not a {data.ndim}D object")
         return SinoNormalizationTask.compute_stats(roi_area)
 
     @staticmethod
@@ -339,7 +334,7 @@ class SinoNormalizationTask(
             elif data.ndim in (0, 1):
                 res = data
             else:
-                raise ValueError("dataset dimension not handled ({})".format(data.ndim))
+                raise ValueError(f"dataset dimension not handled ({data.ndim}D)")
             results[calc_fct.value] = res
         return results
 

tomwer/core/process/reconstruction/normalization/params.py

@@ -34,11 +34,11 @@ __license__ = "MIT"
 __date__ = "25/06/2021"
 
 
+import typing
+
 from silx.utils.enum import Enum as _Enum
 from tomoscan.normalization import Method
 
-import typing
-
 
 class _ValueSource(_Enum):
     MONITOR = "intensity monitor"

tomwer/core/process/reconstruction/saaxis/__init__.py

@@ -1,2 +1,2 @@
-from .saaxis import SAAxisProcess  # noqa F401
 from .params import SAAxisParams  # noqa F401
+from .saaxis import SAAxisProcess  # noqa F401

tomwer/core/process/reconstruction/saaxis/params.py

@@ -30,9 +30,11 @@ __date__ = "18/02/2021"
 
 from collections.abc import Iterable
 from typing import Union
+
+import numpy
 from silx.utils.enum import Enum as _Enum
+
 from tomwer.core.process.reconstruction.scores.params import SABaseParams
-import numpy
 
 
 class ReconstructionMode(_Enum):
@@ -92,7 +94,7 @@ class SAAxisParams(SABaseParams):
     @image_width.setter
     def image_width(self, width):
         if not isinstance(width, (type(None), float, int)):
-            raise TypeError("None, int or float expected. Not {}".format(type(width)))
+            raise TypeError(f"None, int or float expected. Not {type(width)}")
         else:
             self._image_width = width
 
@@ -136,12 +138,9 @@ class SAAxisParams(SABaseParams):
         if self.slice_indexes is None:
             missing_information.append("slice index not provided")
         if len(missing_information) > 0:
+            missing_information_str = " ; ".join(missing_information)
             raise ValueError(
-                str(
-                    "Some informations are missing: {}".format(
-                        " ; ".join(missing_information)
-                    )
-                )
+                f"Some informations are missing: {missing_information_str}"
             )
 
     @staticmethod