processview 0.1.0b0__tar.gz → 0.1.2__tar.gz

home/payno/.local/share/virtualenvs/tomwer_venv/lib/python3.7/site-packages/processview/core/dataset.py

@@ -28,13 +28,30 @@ __license__ = "MIT"
 __date__ = "29/01/2021"
 
 
+class DatasetIdentifier:
+    def __init__(self, dataset):
+        if not isinstance(dataset, Dataset):
+            raise TypeError("{} should be an instance of Dataset".format(dataset))
+        self._dataset_builder = dataset.from_dataset_identifier
+
+    def recreate_dataset(self):
+        """Recreate the dataset from the identifier"""
+        return self._dataset_builder(self)
+
+    def long_description(self) -> str:
+        """long description of the identifier"""
+        return ""
+
+
 class Dataset:
     """Base class that class processes should inherit"""
 
-    def short_str(self):
-        """short string description"""
-        raise NotImplementedError()
+    @staticmethod
+    def from_dataset_identifier(identifier):
+        """Return the Dataset from a identifier"""
+        raise NotImplementedError("Base class")
 
-    def __str__(self):
-        """string representation"""
-        raise NotImplementedError()
+    def get_dataset_identifier(self) -> DatasetIdentifier:
+        """dataset unique identifier. Can be for example a hdf5 and
+        en entry from which the dataset can be rebuild"""
+        raise NotImplementedError("Base class")

home/payno/.local/share/virtualenvs/tomwer_venv/lib/python3.7/site-packages/processview/core/manager/manager.py

@@ -30,6 +30,7 @@ __date__ = "03/11/2020"
 
 from processview.utils.singleton import singleton
 from processview.core.dataset import Dataset
+from processview.core.dataset import DatasetIdentifier
 from collections import OrderedDict
 from silx.utils.enum import Enum as _Enum
 import threading
@@ -60,7 +61,7 @@ class ProcessManager:
     def __init__(self):
         self._processes = {}
         self._dataset_process_states = {}
-        """key is Dataset.short_str. Value is a tuple of
+        """key is processID, Value is a tuple of
         (dataset state, details)"""
         self._processID = 0
         self._updateCallback = set()
@@ -69,6 +70,15 @@ class ProcessManager:
         """list of callback to trigger when a process is added"""
         self.lock = threading.Lock()
 
+    def met(self, process, dataset) -> bool:
+        """check if a dataset has already met a dataset"""
+        if isinstance(dataset, DatasetIdentifier):
+            dataset_id = dataset
+        else:
+            dataset_id = dataset.get_dataset_identifier()
+        if process.process_id in self._dataset_process_states:
+            return dataset_id in self._dataset_process_states[process.process_id]
+
     def register(self, process) -> int:
         """
         Register a process to the manager
@@ -82,20 +92,17 @@ class ProcessManager:
 
         for callback in self._newProcessCallback:
             callback()
-
         return process_id
 
-    def _find_dataset(self, data):
+    def _find_dataset_id(self, data) -> Dataset:
         if isinstance(data, Dataset):
+            return data.get_dataset_identifier()
+        elif isinstance(data, DatasetIdentifier):
             return data
-        elif isinstance(data, str):
-            for s in self.get_datasets():
-                if s == data:
-                    return data
         else:
             raise TypeError(
                 "dataset should be an instance of Dataset or "
-                "str. Get {} instead".format(type(data))
+                "DatasetIdentifier. Get {} instead".format(type(data))
             )
 
     def _find_process(self, process):
@@ -106,7 +113,7 @@ class ProcessManager:
         elif isinstance(process, str):
             for p in self.get_processes():
                 if p.name == process:
-                    return process
+                    return p
         else:
             raise TypeError(
                 "process should be an instance of SuperviseProcess or"
@@ -119,7 +126,7 @@ class ProcessManager:
 
         :param BaseProcess process:
         """
-        if process.process_id in self._processes:
+        if process.process_id in self._processes and process.is_master_process:
             del self._processes[process.process_id]
 
     def get_processes(self) -> tuple:
@@ -160,77 +167,73 @@ class ProcessManager:
             self._dataset_process_states[process.process_id] = OrderedDict()
         if details is None:
             details = ""
-        self._dataset_process_states[process.process_id][dataset.short_str()] = (
+        dataset_id = dataset.get_dataset_identifier()
+        self._dataset_process_states[process.process_id][dataset_id] = (
             state,
             datetime.now(),
             details,
         )
         self.updated()
 
-    def get_dataset_state(self, dataset, process) -> Union[None, DatasetState]:
+    def get_dataset_state(self, dataset_id, process) -> Union[None, DatasetState]:
         """
 
-        :param Dataset dataset:
+        :param Dataset dataset_id:
         :param BaseProcess process:
         :return: DatasetState relative to provided process if know
         :rtype: Union[None, DatasetState]
         """
-        dataset = self._find_dataset(dataset)
+        dataset_id = self._find_dataset_id(dataset_id)
+        assert isinstance(dataset_id, DatasetIdentifier)
         process = self._find_process(process)
         if process is None:
             _logger.warning("process {} is no more supervised".format(process))
             return
-        if dataset is None:
-            _logger.warning("dataset {} is no more supervised".format(dataset))
-        if process.process_id in self._dataset_process_states:
-            if dataset.short_str() in self._dataset_process_states[process.process_id]:
-                return self._dataset_process_states[process.process_id][
-                    dataset.short_str()
-                ][0]
+        if dataset_id is None:
+            _logger.warning("dataset {} is no more supervised".format(dataset_id))
+        elif process.process_id in self._dataset_process_states:
+            if dataset_id in self._dataset_process_states[process.process_id]:
+                return self._dataset_process_states[process.process_id][dataset_id][0]
         return None
 
-    def get_dataset_details(self, dataset, process) -> Union[None, str]:
+    def get_dataset_details(self, dataset_id, process) -> Union[None, str]:
         """
 
-        :param Dataset dataset:
+        :param Dataset dataset_id:
         :param BaseProcess process:
         :return: DatasetState relative to provided process if know
         :rtype: Union[None, DatasetState]
         """
-        dataset = self._find_dataset(dataset)
+        dataset_id = self._find_dataset_id(dataset_id)
         process = self._find_process(process)
         if process is None:
             _logger.warning("process {} is no more supervised".format(process))
             return
-        if dataset is None:
-            _logger.warning("dataset {} is no more supervised".format(dataset))
+        if dataset_id is None:
+            _logger.warning("dataset {} is no more supervised".format(dataset_id))
         if process.process_id in self._dataset_process_states:
-            if dataset.short_str() in self._dataset_process_states[process.process_id]:
-                return self._dataset_process_states[process.process_id][
-                    dataset.short_str()
-                ][2]
+            if dataset_id in self._dataset_process_states[process.process_id]:
+                return self._dataset_process_states[process.process_id][dataset_id][2]
         return None
 
-    def get_dataset_time_stamp(self, dataset, process) -> Union[None, str]:
+    def get_dataset_time_stamp(self, dataset_id, process) -> Union[None, str]:
         """
 
-        :param Dataset dataset:
+        :param Dataset dataset_id:
         :param BaseProcess process:
         :return: DatasetState relative to provided process if know
         :rtype: Union[None, DatasetState]
         """
-        dataset = self._find_dataset(dataset)
+        dataset_id = self._find_dataset_id(dataset_id)
         process = self._find_process(process)
         if process is None:
             _logger.warning("process {} is no more supervised".format(process))
             return
-        if dataset is None:
-            _logger.warning("dataset {} is no more supervised".format(dataset))
+        if dataset_id is None:
+            _logger.warning("dataset {} is no more supervised".format(dataset_id))
         if process.process_id in self._dataset_process_states:
-            if dataset.short_str() in self._dataset_process_states[process.process_id]:
-                return self._dataset_process_states[process.process_id][
-                    dataset.short_str()
-                ][1]
+            if dataset_id in self._dataset_process_states[process.process_id]:
+                return self._dataset_process_states[process.process_id][dataset_id][1]
         return None
 
     def get_dataset_stream(self, dataset, time_stamp=False) -> tuple:
@@ -244,8 +247,9 @@ class ProcessManager:
         """
         stream = []
         for process_id, dataset_states in self._dataset_process_states.items():
-            if dataset.short_str() in dataset_states:
-                state, _timestamp, _ = dataset_states[dataset.short_str()]
+            dst_id = dataset.get_dataset_identifier()
+            if dst_id in dataset_states:
+                state, _timestamp, _ = dataset_states[dst_id]
                 stream.append((process_id, state, _timestamp))
         # order the stream
         stream = sorted(stream, key=lambda elmt: elmt[2])
@@ -260,7 +264,7 @@ class ProcessManager:
         :param BaseProcess process:
         :param bool time_stamp: if True then return timestamp in the list of
                                 elements
-        :return: tuple of (dataset id, state, [timestamp])
+        :return: tuple of (DatasetIdentifier, state, [timestamp])
         :rtype: tuple
         """
         history = []

home/payno/.local/share/virtualenvs/tomwer_venv/lib/python3.7/site-packages/processview/core/manager/test/test_manager.py

@@ -33,6 +33,7 @@ from processview.core.superviseprocess import SuperviseProcess
 from processview.core.manager import ProcessManager
 from processview.core.manager import DatasetState
 from processview.core.dataset import Dataset
+from processview.core.dataset import DatasetIdentifier
 import gc
 
 
@@ -81,10 +82,11 @@ class TestProcessManager(unittest.TestCase):
         )
         # test 'getScanState'
         self.assertEqual(
-            manager.get_dataset_state(dataset=scan_2, process=p2), DatasetState.FAILED
+            manager.get_dataset_state(dataset_id=scan_2, process=p2),
+            DatasetState.FAILED,
         )
 
-        self.assertEqual(manager.get_dataset_state(dataset=scan_1, process=p2), None)
+        self.assertEqual(manager.get_dataset_state(dataset_id=scan_1, process=p2), None)
 
         # test 'getScanStream'
         self.assertEqual(
@@ -98,7 +100,10 @@ class TestProcessManager(unittest.TestCase):
         # test 'getProcessHistory'
         self.assertEqual(
             manager.get_process_history(process=p1),
-            ((str(scan_1), DatasetState.PENDING), (str(scan_2), DatasetState.SUCCEED)),
+            (
+                (scan_1.get_dataset_identifier(), DatasetState.PENDING),
+                (scan_2.get_dataset_identifier(), DatasetState.SUCCEED),
+            ),
         )
 
 
@@ -110,9 +115,24 @@ class _DummyScan(Dataset):
     def __str__(self) -> str:
         return self.name
 
-    def short_str(self):
+    def get_dataset_identifier(self) -> DatasetIdentifier:
+        return _DummyIdentifier(self)
+
+
+class _DummyIdentifier(DatasetIdentifier):
+    def __init__(self, dataset):
+        super().__init__(dataset)
+        self.name = dataset.name
+
+    def __str__(self):
         return self.name
 
+    def __eq__(self, other):
+        return self.name == other.name
+
+    def __hash__(self):
+        return hash(self.name)
+
 
 def suite():
     test_suite = unittest.TestSuite()

home/payno/.local/share/virtualenvs/tomwer_venv/lib/python3.7/site-packages/processview/core/superviseprocess.py

@@ -63,6 +63,9 @@ class SuperviseProcess:
 
     def __init__(self, name=None, process_id=None):
         self._name = name
+        self._is_master_process = process_id is None
+        """is this process the master process of the SuperviseProcess. Or is
+        it just one of the underlying process of a master SuperviseProcess"""
         if process_id is None:
             ProcessManager().register(self)
         else:
@@ -71,6 +74,10 @@ class SuperviseProcess:
     def __del__(self):
         ProcessManager().unregister(self)
 
+    @property
+    def is_master_process(self):
+        return self._is_master_process
+
     @property
     def process_id(self):
         return self.__process_id
@@ -114,3 +121,11 @@ class SuperviseProcess:
         ProcessManager().notify_dataset_state(
             dataset=dataset, state=state, process=self, details=details
         )
+
+    def reprocess(self, dataset):
+        """
+        Reprocess a dataset.
+
+        :param Dataset: dataset to reprocess
+        """
+        raise NotImplementedError("Base class")