scipion-pyworkflow 3.10.0__py3-none-any.whl → 3.10.2__py3-none-any.whl

pyworkflow/constants.py

@@ -43,7 +43,7 @@ VERSION_1 = '1.0.0'
 VERSION_1_1 = '1.1.0'
 VERSION_1_2 = '1.2.0'
 VERSION_2_0 = '2.0.0'
-VERSION_3_0 = '3.10.0'
+VERSION_3_0 = '3.10.2'
 
 # For a new release, define a new constant and assign it to LAST_VERSION
 # The existing one has to be added to OLD_VERSIONS list.

pyworkflow/protocol/executor.py

@@ -144,9 +144,9 @@ class StepExecutor:
 
 class StepThread(threading.Thread):
     """ Thread to run Steps in parallel. """
-    def __init__(self, thId, step, lock):
+    def __init__(self, step, lock):
         threading.Thread.__init__(self)
-        self.thId = thId
+        self.thId = step.getObjId()
         self.step = step
         self.lock = lock
 
@@ -262,7 +262,7 @@ class ThreadStepExecutor(StepExecutor):
                 return []
             else:
                 return gpus
-    def getFreeGpuSlot(self, nodeId=None):
+    def getFreeGpuSlot(self, stepId=None):
         """ Returns a free gpu slot available or None. If node is passed it also reserves it for that node
 
         :param node: node to make the reserve of Gpus
@@ -272,10 +272,10 @@ class ThreadStepExecutor(StepExecutor):
             if node < 0:
                 gpus = self.gpuDict[node]
 
-                if nodeId is not None:
+                if stepId is not None:
                     self.gpuDict.pop(node)
-                    self.gpuDict[nodeId] = gpus
-                    logger.info("GPUs %s assigned to thread %s" % (gpus, nodeId))
+                    self.gpuDict[stepId] = gpus
+                    logger.info("GPUs %s assigned to step %s" % (gpus, stepId))
                 else:
                     logger.info("Free gpu slot found at %s" % node)
                 return gpus
@@ -288,14 +288,14 @@ class ThreadStepExecutor(StepExecutor):
         if gpus is not None:
             self.gpuDict.pop(node)
             self.gpuDict[-node-1] = gpus
-            logger.info("GPUs %s freed from thread %s" % (gpus, node))
+            logger.info("GPUs %s freed from step %s" % (gpus, node))
         else:
-            logger.debug("node %s not found in GPU slots" % node)
+            logger.debug("step id %s not found in GPU slots" % node)
 
     def _isStepRunnable(self, step):
         """ Overwrite this method to check GPUs availability"""
 
-        if self.gpuList and step.needsGPU() and self.getFreeGpuSlot() is None:
+        if self.gpuList and step.needsGPU() and self.getFreeGpuSlot(step.getObjId()) is None:
             logger.info("Can't run step %s. Needs gpus and there are no free gpu slots" % step)
             return False
 
@@ -337,7 +337,7 @@ class ThreadStepExecutor(StepExecutor):
             for node in nodesFinished:
                 step = runningSteps.pop(node)  # remove entry from runningSteps
                 freeNodes.append(node)  # the node is available now
-                self.freeGpusSlot(node)
+                self.freeGpusSlot(step.getObjId())
                 # Notify steps termination and check if we should continue
                 doContinue = stepFinishedCallback(step)
                 if not doContinue:
@@ -361,7 +361,7 @@ class ThreadStepExecutor(StepExecutor):
                         node = freeNodes.pop(0)  # take an available node
                         runningSteps[node] = step
                         logger.debug("Running step %s on node %s" % (step, node))
-                        t = StepThread(node, step, sharedLock)
+                        t = StepThread(step, sharedLock)
                         # won't keep process up if main thread ends
                         t.daemon = True
                         t.start()

pyworkflow/protocol/protocol.py

@@ -362,6 +362,18 @@ class Protocol(Step):
     # prerequisites of steps, but is easier to keep it
     stepsExecutionMode = STEPS_SERIAL
 
+    @classmethod
+    def modeSerial(cls):
+        """ Returns true if steps are run one after another"""
+        # Maybe this property can be inferred from the
+        # prerequisites of steps, but is easier to keep it
+        return cls.stepsExecutionMode == STEPS_SERIAL
+
+    @classmethod
+    def modeParallel(cls):
+        """ Returns true if steps are run in parallel"""
+        return not cls.modeSerial()
+
     def __init__(self, **kwargs):
         Step.__init__(self, **kwargs)
         self._size = None
@@ -1316,7 +1328,7 @@ class Protocol(Step):
                                         prot_id=self.getObjId(),
                                         prot_name=self.getClassName(),
                                         step_id=step._index))
-        if step.isFailed() and self.stepsExecutionMode == STEPS_PARALLEL:
+        if step.isFailed() and self.modeParallel():
             # In parallel mode the executor will exit to close
             # all working threads, so we need to close
             self._endRun()
@@ -2434,7 +2446,7 @@ def runProtocolMain(projectPath, protDbPath, protId):
     executor = None
     nThreads = max(protocol.numberOfThreads.get(), 1)
 
-    if protocol.stepsExecutionMode == STEPS_PARALLEL and nThreads > 1:
+    if protocol.modeParallel() and nThreads > 1:
         if protocol.useQueueForSteps():
             executor = QueueStepExecutor(hostConfig,
                                          protocol.getSubmitDict(),
@@ -2531,10 +2543,7 @@ class ProtStreamingBase(Protocol):
     Minimum number of threads is 3 and should run in parallel mode.
     """
 
-    def __init__(self, **kwargs):
-
-        super().__init__()
-        self.stepsExecutionMode = STEPS_PARALLEL
+    stepsExecutionMode = STEPS_PARALLEL
     def _insertAllSteps(self):
         # Insert the step that generates the steps
         self._insertFunctionStep(self.resumableStepGeneratorStep, str(datetime.now()), needsGPU=False)

{scipion_pyworkflow-3.10.0.dist-info → scipion_pyworkflow-3.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scipion-pyworkflow
-Version: 3.10.0
+Version: 3.10.2
 Summary: Simple workflow platform used in scientific applications, initially developed within the Scipion framework for image processing in Electron Microscopy.
 Home-page: https://github.com/scipion-em/scipion-pyworkflow
 Author: J.M. De la Rosa Trevin, Roberto Marabini, Grigory Sharov, Josue Gomez Blanco, Pablo Conesa, Yunior Fonseca Reyna

{scipion_pyworkflow-3.10.0.dist-info → scipion_pyworkflow-3.10.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
 pyworkflow/__init__.py,sha256=Wr-MVKMQJy_Cy-rpPPB0-pyv8-8tx7GPPaLSNBrV1AI,1246
 pyworkflow/config.py,sha256=_sq6YCRfcfE4WUNXLShgbICbBeTSLpD2KmPVapR1Wos,22306
-pyworkflow/constants.py,sha256=R9MnxV0n3oPb4pwacP6L80gj6inmkuP7yMS572s6OHQ,7448
+pyworkflow/constants.py,sha256=GG3_xDU181oUFld0DkGbvKq5lN606DjaCLnOsuVMDDQ,7448
 pyworkflow/exceptions.py,sha256=3VFxuNJHcIWxRnLPR0vYg0RFAQMmxPBJZLZSi87VI8E,507
 pyworkflow/object.py,sha256=yX3tBUuBssCI8aWGlTmyEKzFE59dJq_7NqllouVzMk0,55069
 pyworkflow/plugin.py,sha256=JJm5plPyOnPAR4To6I5rXVIBda-1Dg-53zicdnLSrac,28858
@@ -67,12 +67,12 @@ pyworkflow/project/scripts/stop.py,sha256=vCeCxkwPCoUkLbna5HCxKWJ1hrsI4U19Sg9JD4
 pyworkflow/protocol/__init__.py,sha256=bAdIpvUW4GAYdIuv92DZ44-OEkZ7lTtnp1S9T5cwtVs,1413
 pyworkflow/protocol/bibtex.py,sha256=mCUk1Hp5Vp_i2lozDM1BQNOw10e_RSu86oXvrR63sOA,2122
 pyworkflow/protocol/constants.py,sha256=DfuCs7eub-mLHJjEpHlIG9BW3fUpRwfTVwMYytNWv6U,3392
-pyworkflow/protocol/executor.py,sha256=OxnRGZeJI8vYJ6ZfSWifN043pvdw1vV_x9sjQd57fi8,17745
+pyworkflow/protocol/executor.py,sha256=x3v1q4NyF--iLMOMS2qsVpOEy-nge9IDuzWLllx5Tis,17769
 pyworkflow/protocol/hosts.py,sha256=B9ENNclqYe75CPqAMOoPjwn-r3ST6HxTewXtsK_zWks,10453
 pyworkflow/protocol/launch.py,sha256=7WKAiHma2tSuhqK4xVnxD_SnVt7Y5qyDFdQwTo8BLF0,11267
 pyworkflow/protocol/package.py,sha256=L6x3HHKtbrhDQRJHD07SG3DQKNMGaRQ0ROoLEY3SuRQ,1444
 pyworkflow/protocol/params.py,sha256=gP6QImgULvzCr-f0iseArIp1bjXm1JuUr9padmuMs7M,25901
-pyworkflow/protocol/protocol.py,sha256=pwgaQzcnuJOkpBDJ8k7ZEc0DJ6m_BrZz627HvWZoW2E,97145
+pyworkflow/protocol/protocol.py,sha256=K-srKr4D220JLwYcDfAAlfBsAvrqf55gnyCZsSTt3uc,97442
 pyworkflow/resources/Imagej.png,sha256=nU2nWI1wxZB_xlOKsZzdUjj-qiCTjO6GwEKYgZ5Risg,14480
 pyworkflow/resources/chimera.png,sha256=AKCuwMqmZo0Cg2sddMUjBWUhmAq-nPsAVCBpVrYNeiQ,815
 pyworkflow/resources/fa-exclamation-triangle_alert.png,sha256=31_XvRu0CkJ2dvHSpcBAR43378lIJTWwiag_A7SuUQc,585
@@ -131,10 +131,10 @@ pyworkflowtests/tests/test_protocol_export.py,sha256=z18nKPkOnrYLMU8KqcnVsF6-ylQ
 pyworkflowtests/tests/test_protocol_output.py,sha256=8gnIFMRNmwPnIBRCG29WHJB6mqK4FLGn1jiXHtTD6pY,5980
 pyworkflowtests/tests/test_streaming.py,sha256=vOH-bKCM-fVUSsejqNnCX5TPXhdUayk9ZtJHsNVcfCY,1615
 pyworkflowtests/tests/test_utils.py,sha256=_pTYGCuXC7YNMdCBzUYNfSBCR3etrHsxHfIhsQi4VPc,7465
-scipion_pyworkflow-3.10.0.dist-info/LICENSE.txt,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
-scipion_pyworkflow-3.10.0.dist-info/METADATA,sha256=VMXHL5VKBHV-p3EyaKLc1yk_E7tUVSUkH2bQSBnLX7w,4682
-scipion_pyworkflow-3.10.0.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-scipion_pyworkflow-3.10.0.dist-info/dependency_links.txt,sha256=D7r_CPRjYRtBb3q_OBocTdsaeXI5TwnYMu5ri0JFtzs,84
-scipion_pyworkflow-3.10.0.dist-info/entry_points.txt,sha256=oR-zwsOICjEPINm-FWVPp-RfnpXZanVal4_XG6BWkkQ,127
-scipion_pyworkflow-3.10.0.dist-info/top_level.txt,sha256=PzyJteyenJwLjAeSFP7oYrTN_U71GABQwET8oLZkh9k,27
-scipion_pyworkflow-3.10.0.dist-info/RECORD,,
+scipion_pyworkflow-3.10.2.dist-info/LICENSE.txt,sha256=ixuiBLtpoK3iv89l7ylKkg9rs2GzF9ukPH7ynZYzK5s,35148
+scipion_pyworkflow-3.10.2.dist-info/METADATA,sha256=-H5H84If9BMyHHX3bBRSZGwyJurbKwmlsUOHyi8U3_Y,4682
+scipion_pyworkflow-3.10.2.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+scipion_pyworkflow-3.10.2.dist-info/dependency_links.txt,sha256=D7r_CPRjYRtBb3q_OBocTdsaeXI5TwnYMu5ri0JFtzs,84
+scipion_pyworkflow-3.10.2.dist-info/entry_points.txt,sha256=oR-zwsOICjEPINm-FWVPp-RfnpXZanVal4_XG6BWkkQ,127
+scipion_pyworkflow-3.10.2.dist-info/top_level.txt,sha256=PzyJteyenJwLjAeSFP7oYrTN_U71GABQwET8oLZkh9k,27
+scipion_pyworkflow-3.10.2.dist-info/RECORD,,