tomwer 1.1.0rc7__tar.gz → 1.2.0a0__tar.gz

tomwer-1.2.0a0/PKG-INFO

@@ -0,0 +1,177 @@
+Metadata-Version: 2.1
+Name: tomwer
+Version: 1.2.0a0
+Summary: "tomography workflow tools"
+Home-page: https://gitlab.esrf.fr/tomotools/tomwer
+Author: data analysis unit
+Author-email: henri.payno@esrf.fr
+License: MIT
+Project-URL: Bug Tracker, https://gitlab.esrf.fr/tomotools/tomwer/-/issues
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Environment :: Console
+Classifier: Environment :: X11 Applications :: Qt
+Classifier: Operating System :: POSIX
+Classifier: Natural Language :: English
+Classifier: Topic :: Scientific/Engineering :: Physics
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Provides-Extra: doc
+Provides-Extra: _full_base
+Provides-Extra: full_no_cuda
+Provides-Extra: full
+Provides-Extra: _dev_spec
+Provides-Extra: dev
+Provides-Extra: dev_no_cuda
+Provides-Extra: setup_requires
+License-File: LICENSE
+
+tomwer
+======
+
+tomwer is offering tools to automate acquisition and reconstruction processes for Tomography.
+It contains:
+
+- a library to access each acquisition process individually
+- gui and applications to control main processes (reconstruction, data transfert...) and execute them as a stand alone application.
+- an orange add-on to help users defining their own workflow (http://orange.biolab.si)
+
+
+
+.. image:: http://www.edna-site.org/pub/doc/tomwer/extra/tomwer_start_short.gif
+
+
+.. |Gitlab Status| image:: https://gitlab.esrf.fr/tomotools/tomwer/badges/master/pipeline.svg
+    :target: https://gitlab.esrf.fr/tomotools/tomwer/pipelines
+
+
+Documentation
+-------------
+
+Documentation of latest release is available at http://www.edna-site.org/pub/doc/tomwer/latest
+
+Installation
+------------
+
+Step 1 - Create a virtual env (recommended)
+'''''''''''''''''''''''''''''''''''''''''''
+
+It is recommended to create a python virtual environment to run the workflow tool.
+Virtual environment might avoid some conflict between python packages. But you can also install it on your 'current' python environment and move to step 1.
+
+.. code-block:: bash
+
+   virtualenv --python=python3 --system-site-packages myvirtualenv
+
+
+Then activate the virtual environment
+
+.. code-block:: bash
+
+   source myvirtualenv/bin/activate
+
+First update pip and setuptools to avoid some potential errors
+
+.. code-block:: bash
+
+   pip install --upgrade pip
+   pip install setuptools --upgrade
+
+
+.. note:: To quit the virtual environment
+
+   .. code-block:: bash
+
+      deactivate
+
+Step 2 - tomwer (madatory)
+''''''''''''''''''''''''''
+
+To install it with all 'features':
+
+.. code-block:: bash
+
+    pip install tomwer[full]
+
+alternatively you can install the master branch from
+
+.. code-block:: bash
+
+    pip install git+https://gitlab.esrf.fr/tomotools/tomwer/#egg=tomwer[full]
+
+
+Step 3 - update orange-canvas-core and orange-widget-base (recommended)
+'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
+
+To access 'processing' wheels you might want to install forks of update orange-canvas-core and orange-widget-base
+
+.. code-block:: bash
+
+    pip install https://github.com/payno/orange-canvas-core --no-deps --upgrade
+    pip install https://github.com/payno/orange-widget-base --no-deps --upgrade
+
+
+Launching applications
+::::::::::::::::::::::
+
+After the installation tomwer is embedding several applications.
+
+Those applications can be launched by calling:
+
+.. code-block:: bash
+
+   tomwer appName {options}
+
+.. note:: if you only call `tomwer` then the man page will be displayed.
+
+.. note:: You can access each application help using ``
+
+    .. code-block:: bash
+
+       tomwer appName --help
+
+
+tomwer canvas - orange canvas
+'''''''''''''''''''''''''''''
+
+You can launch the canvas to create workflows from the different 'bricks'
+
+.. code-block:: bash
+
+   tomwer canvas
+
+.. note:: you can also use `orange-canvas`
+
+.. note:: if your installed a virtual environment do not forget to active it :
+
+    .. code-block:: bash
+
+       source myvirtualenv/bin/activate
+
+
+Documentation
+:::::::::::::
+
+.. code-block:: bash
+
+   cd doc
+   make html
+
+The documentation is build in doc/build/html and the entry point is index.html
+
+.. code-block:: bash
+
+   firefox build/html/index.html
+
+.. note:: the build of the documentation need sphinx to be installed. This is not an hard dependacy. So you might need to install it.
+
+
+You also should generate documentation to be accessible from Orange GUI (pressing the F1 key).
+
+.. code-block:: bash
+
+   cd doc
+   make htmlhelp

tomwer-1.2.0a0/orangecontrib/tomwer/__init__.py

@@ -0,0 +1,6 @@
+import logging
+
+from . import state_summary  # noqa F401
+
+fabio_logger = logging.getLogger("fabio.edfimage")
+fabio_logger.setLevel(logging.WARNING)

tomwer-1.2.0a0/orangecontrib/tomwer/orange/managedprocess.py

@@ -0,0 +1,114 @@
+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["H. Payno"]
+__license__ = "MIT"
+__date__ = "22/20/2020"
+
+
+import functools
+
+from ewoksorange.bindings import OWEwoksWidgetWithTaskStack
+from ewoksorange.bindings.owwidgets import invalid_data
+from orangewidget.widget import OWBaseWidget
+from processview.core.manager import DatasetState, ProcessManager
+from processview.core.superviseprocess import SuperviseProcess
+
+from orangecontrib.tomwer.widgets.utils import WidgetLongProcessing
+
+
+class _SuperviseMixIn(SuperviseProcess):
+    def __init__(self, process_id=None):
+        SuperviseProcess.__init__(self, process_id=process_id)
+        self.destroyed.connect(functools.partial(ProcessManager().unregister, self))
+
+    def setCaption(self, caption):
+        self.name = caption
+
+    def notify_skip(self, scan, details=None):
+        ProcessManager().notify_dataset_state(
+            dataset=scan, process=self, state=DatasetState.SKIPPED, details=details
+        )
+
+    def notify_pending(self, scan, details=None):
+        ProcessManager().notify_dataset_state(
+            dataset=scan, process=self, state=DatasetState.PENDING, details=details
+        )
+
+    def notify_succeed(self, scan, details=None):
+        ProcessManager().notify_dataset_state(
+            dataset=scan, process=self, state=DatasetState.SUCCEED, details=details
+        )
+
+    def notify_failed(self, scan, details=None):
+        ProcessManager().notify_dataset_state(
+            dataset=scan, process=self, state=DatasetState.FAILED, details=details
+        )
+
+    def notify_on_going(self, scan, details=None):
+        ProcessManager().notify_dataset_state(
+            dataset=scan, process=self, state=DatasetState.ON_GOING, details=details
+        )
+
+
+class SuperviseOW(OWBaseWidget, _SuperviseMixIn, openclass=True):
+    """
+    A basic OWWidget but registered on the process manager
+    """
+
+    want_control_area = False
+
+    def __init__(self, parent, process_id=None):
+        OWBaseWidget.__init__(self, parent, process_id=process_id)
+        _SuperviseMixIn.__init__(self, process_id=process_id)
+
+    def setCaption(self, caption):
+        OWBaseWidget.setCaption(self, caption)
+        _SuperviseMixIn.setCaption(self, caption=caption)
+
+
+class TomwerWithStackStack(
+    OWEwoksWidgetWithTaskStack, _SuperviseMixIn, WidgetLongProcessing, openclass=True
+):
+    def __init__(self, parent, process_id=None, *args, **kwargs):
+        OWEwoksWidgetWithTaskStack.__init__(self, parent, args, kwargs)
+        _SuperviseMixIn.__init__(self, process_id=process_id)
+
+        self.task_executor_queue.sigComputationStarted.connect(self._startProcessing)
+        self.task_executor_queue.sigComputationEnded.connect(self._endProcessing)
+
+    def setCaption(self, caption):
+        OWBaseWidget.setCaption(self, caption)
+        _SuperviseMixIn.setCaption(self, caption=caption)
+
+    def trigger_downstream(self) -> None:
+        # for now ewoksorange send ewoks variable. This will work only if
+        # all task are implemented using ewokwidget which is not the case today
+        for ewoksname, var in self.get_task_outputs().items():
+            channel = self._get_output_signal(ewoksname)
+            if invalid_data.is_invalid_data(var.value):
+                channel.send(None)  # or channel.invalidate?
+            else:
+                channel.send(var.value)

tomwer-1.2.0a0/orangecontrib/tomwer/state_summary.py

@@ -0,0 +1,50 @@
+from orangewidget.utils.signals import PartialSummary, summarize
+
+from tomwer.core.cluster import SlurmClusterConfiguration
+from tomwer.core.futureobject import FutureTomwerObject
+from tomwer.core.scan.blissscan import BlissScan
+from tomwer.core.scan.scanbase import TomwerScanBase
+
+
+@summarize.register(object)  # noqa F811
+def summarize_(Object: object):  # noqa F811
+    return PartialSummary("any object", "an oject of any type")
+
+
+@summarize.register(dict)  # noqa F811
+def summarize_(configuration: dict):  # noqa F811
+    return PartialSummary(
+        "any configuration", "any configuration that can be provided to a process"
+    )
+
+
+@summarize.register(SlurmClusterConfiguration)  # noqa F811
+def summarize_(cluster_config: SlurmClusterConfiguration):  # noqa F811
+    return PartialSummary(
+        "cluster configuration",
+        "cluster configuration to launch some remote processing",
+    )
+
+
+@summarize.register(TomwerScanBase)  # noqa F811
+def summarize_(data: TomwerScanBase):  # noqa F811
+    return PartialSummary(
+        "dataset with processing history",
+        "core object used to ship dataset and history of processing done on this dataset",
+    )
+
+
+@summarize.register(FutureTomwerObject)  # noqa F811
+def summarize_(future_data: FutureTomwerObject):  # noqa F811
+    return PartialSummary(
+        "dataset with pending processing",
+        "object used when there is some pending processing (asyncio.future). Can be convert back to `data`",
+    )
+
+
+@summarize.register(BlissScan)  # noqa F811
+def summarize_(bliss_scan: BlissScan):  # noqa F811
+    return PartialSummary(
+        "raw dataset from bliss",
+        "object used when debug some processing relative to bliss",
+    )

tomwer-1.2.0a0/orangecontrib/tomwer/widgets/__init__.py

@@ -0,0 +1,62 @@
+# coding: utf-8
+###########################################################################
+# Copyright (C) 2016 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+#############################################################################
+
+import sysconfig
+
+import pkg_resources
+
+ICON = "../widgets/icons/tomwer.png"
+
+BACKGROUND = "#C0CCFF"
+
+
+# Entry point for main Orange categories/widgets discovery
+def widget_discovery(discovery):
+    dist = pkg_resources.get_distribution("tomwer")
+    pkgs = [
+        "orangecontrib.tomwer.widgets.cluster",
+        "orangecontrib.tomwer.widgets.control",
+        "orangecontrib.tomwer.widgets.debugtools",
+        "orangecontrib.tomwer.widgets.edit",
+        "orangecontrib.tomwer.widgets.reconstruction",
+        "orangecontrib.tomwer.widgets.visualization",
+        "orangecontrib.tomwer.widgets.other",
+    ]
+    for pkg in pkgs:
+        discovery.process_category_package(pkg, distribution=dist)
+
+
+WIDGET_HELP_PATH = (
+    # Used for development.
+    # You still need to build help pages using
+    # make htmlhelp
+    # inside doc folder
+    ("/build/sphinx/htmlhelp/index.html", None),
+    #
+    # Documentation included in wheel
+    # Correct DATA_FILES entry is needed in setup.py and documentation has to be built
+    # # before the wheel is created.
+    (f"{sysconfig.get_path('data')}/help/tomwer/index.html", None),
+    ("http://www.silx.org/pub/doc/tomwer/htmlhelp/0.9/", None),
+)

tomwer-1.2.0a0/orangecontrib/tomwer/widgets/cluster/FutureSupervisorOW.py

@@ -0,0 +1,244 @@
+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["H. Payno"]
+__license__ = "MIT"
+__date__ = "14/10/2021"
+
+
+import time
+
+from orangewidget import gui
+from orangewidget.settings import Setting
+from orangewidget.widget import Input, Output, OWBaseWidget
+from processview.core.manager import DatasetState, ProcessManager
+from processview.core.superviseprocess import SuperviseProcess
+from silx.gui import qt
+from silx.gui.utils.concurrent import submitToQtMainThread
+
+from tomwer.core.futureobject import FutureTomwerObject
+from tomwer.core.scan.scanbase import TomwerScanBase
+from tomwer.core.volume.volumebase import TomwerVolumeBase
+from tomwer.gui.cluster.supervisor import (
+    FutureTomwerScanObserverWidget as _FutureTomwerScanObserverWidget,
+)
+
+
+class FutureSupervisorOW(OWBaseWidget, openclass=True):
+    """
+    Orange widget to define a slurm cluster as input of other
+    widgets (based on nabu for now)
+    """
+
+    name = "future supervisor"
+    id = "orange.widgets.tomwer.cluster.FutureSupervisorOW.FutureSupervisorOW"
+    description = "Observe slurm job registered."
+    icon = "icons/slurmobserver.svg"
+    priority = 22
+    keywords = [
+        "tomography",
+        "tomwer",
+        "slurm",
+        "observer",
+        "cluster",
+        "job",
+        "sbatch",
+        "supervisor",
+        "future",
+    ]
+
+    want_main_area = True
+    want_control_area = False
+    resizing_enabled = True
+    compress_signal = False
+
+    _ewoks_default_inputs = Setting(dict())
+
+    class Inputs:
+        future_in = Input(
+            name="future_tomo_obj",
+            type=FutureTomwerObject,
+            doc="data with some remote processing",
+            multiple=True,
+            default=True,
+        )
+
+    class Outputs:
+        data = Output(name="data", type=TomwerScanBase)
+        volume = Output(name="volume", type=TomwerVolumeBase)
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        # gui
+        layout = gui.vBox(self.mainArea, self.name).layout()
+        self._widget = FutureTomwerObjectObserverWidget(
+            parent=self, name=self.windowTitle()
+        )
+        layout.addWidget(self._widget)
+
+        # connect signal / slot
+        self._widget.observationTable.model().sigStatusUpdated.connect(
+            self._convertBackAutomatically
+        )
+        self._widget.sigConversionRequested.connect(self._convertBack)
+
+    def convertWhenFinished(self):
+        return self._widget.convertWhenFinished()
+
+    def _convertBackAutomatically(self, future_tomo_obj, status):
+        if not isinstance(future_tomo_obj, FutureTomwerObject):
+            raise TypeError(
+                f"future_tomo_obj is expected to be an instance of {FutureTomwerObject} and not {type(future_tomo_obj)}"
+            )
+        if status in ("finished", "completed") and self.convertWhenFinished():
+            self._convertBack(future_tomo_obj)
+
+    def _convertBack(self, future_tomo_obj):
+        if not isinstance(future_tomo_obj, FutureTomwerObject):
+            raise TypeError(
+                f"future_tomo_obj is expected to be an instance of {FutureTomwerObject} and not {type(future_tomo_obj)}"
+            )
+        self._futureHasBeenConverted(future_tomo_obj, future_tomo_obj.tomo_obj)
+
+    @Inputs.future_in
+    def add(self, future_tomo_obj, signal_id=None):
+        if future_tomo_obj is not None:
+            self._widget.addFutureTomoObj(future_tomo_obj=future_tomo_obj)
+
+    def _futureHasBeenConverted(self, future_tomo_obj, tomo_obj):
+        # clean client to free resources
+        self._widget.removeFutureTomoObj(future_tomo_obj=future_tomo_obj)
+        if tomo_obj is not None:
+            if isinstance(tomo_obj, TomwerScanBase):
+                self.Outputs.data.send(tomo_obj)
+            elif isinstance(tomo_obj, TomwerVolumeBase):
+                self.Outputs.volume.send(tomo_obj)
+
+
+class FutureTomwerObjectObserverWidget(
+    _FutureTomwerScanObserverWidget, SuperviseProcess
+):
+    """add dataset state notification (ProcessManager) to the original FutureTomwerScanObserverWidget"""
+
+    REFRESH_FREQUENCE = 10
+    """time between call to updateView"""
+
+    def __init__(self, name, parent=None):
+        super().__init__(parent=parent)
+        self.name = name
+        self._updateThread = _RefreshThread(
+            callback=self.updateView, refresh_frequence=self.REFRESH_FREQUENCE
+        )
+        self.destroyed.connect(self.stopRefresh)
+        self._updateThread.start()
+
+    def stopRefresh(self):
+        if self._updateThread is not None:
+            self._updateThread.stop()
+            self._updateThread.wait(self.REFRESH_FREQUENCE + 1)
+            self._updateThread = None
+
+    def close(self):
+        self.stopRefresh()
+        super().close()
+
+    def addFutureTomoObj(self, future_tomo_obj: FutureTomwerObject):
+        super().addFutureTomoObj(future_tomo_obj)
+        self._updateTomoObjSupervisor(future_tomo_obj)
+
+    def removeFutureTomoObj(self, future_tomo_obj: FutureTomwerObject):
+        self._updateTomoObjSupervisor(future_tomo_obj)
+        super().removeFutureTomoObj(future_tomo_obj)
+
+    def _updateTomoObjSupervisor(self, future_tomo_obj):
+        r_id = future_tomo_obj.process_requester_id
+        if r_id is not None:
+            requester_name = ProcessManager().get_process(r_id).name
+        else:
+            requester_name = "unknow"
+        details = f"job spawn by {requester_name}"
+        if future_tomo_obj is None:
+            return
+        elif future_tomo_obj.status == "error":
+            state = DatasetState.FAILED
+        elif future_tomo_obj.status == "pending":
+            details = "\n".join([details, "pending"])
+            state = DatasetState.PENDING
+        elif future_tomo_obj.status in ("finished", "completed"):
+            details = future_tomo_obj.logs or "no log found"
+            state = DatasetState.SUCCEED
+        elif future_tomo_obj.status == "running":
+            details = "\n".join([details, "running"])
+            state = DatasetState.ON_GOING
+        elif future_tomo_obj.status == "cancelled":
+            details = "\n".join([details, "job cancelled"])
+            state = DatasetState.SKIPPED
+        elif future_tomo_obj.status is None:
+            return
+        else:
+            raise ValueError(
+                f"future scan status '{future_tomo_obj.status}' is not managed, {type(future_tomo_obj.status)}"
+            )
+        ProcessManager().notify_dataset_state(
+            dataset=future_tomo_obj.tomo_obj,
+            process=self,
+            state=state,
+            details=details,
+        )
+
+    def _updateStatus(self, future_tomo_obj):
+        self._updateTomoObjSupervisor(future_tomo_obj)
+        super()._updateStatus(future_tomo_obj)
+
+
+class _RefreshThread(qt.QThread):
+    """Simple thread to call a refresh callback each refresh_frequence (seconds)"""
+
+    TIME_BETWEEN_LOOP = 1.0
+
+    def __init__(self, callback, refresh_frequence) -> None:
+        super().__init__()
+        self._callback = callback
+        self._refresh_frequence = refresh_frequence
+        self._stop = False
+
+    def stop(self):
+        self._stop = True
+        self._callback = None
+
+    def run(self):
+        w_t = self._refresh_frequence + self.TIME_BETWEEN_LOOP
+
+        while not self._stop:
+            if w_t <= 0:
+                if self._callback is not None:
+                    try:
+                        submitToQtMainThread(self._callback)
+                    except AttributeError:
+                        # can happen when closing
+                        pass
+                w_t = self._refresh_frequence + self.TIME_BETWEEN_LOOP
+            w_t -= self.TIME_BETWEEN_LOOP
+            time.sleep(self.TIME_BETWEEN_LOOP)