scipion-pyworkflow 3.7.0__py3-none-any.whl → 3.9.0__py3-none-any.whl

pyworkflow/protocol/protocol.py

@@ -25,8 +25,7 @@
 This modules contains classes required for the workflow
 execution and tracking like: Step and Protocol
 """
-import contextlib
-import sys, os
+import os
 import json
 import threading
 import time
@@ -36,7 +35,7 @@ import pyworkflow as pw
 from pyworkflow.exceptions import ValidationException, PyworkflowException
 from pyworkflow.object import *
 import pyworkflow.utils as pwutils
-from pyworkflow.utils.log import LoggingConfigurator, getExtraLogInfo, STATUS, setDefaultLoggingContext
+from pyworkflow.utils.log import getExtraLogInfo, STATUS, setDefaultLoggingContext
 from .executor import StepExecutor, ThreadStepExecutor, QueueStepExecutor
 from .constants import *
 from .params import Form
@@ -50,21 +49,24 @@ logger = logging.getLogger(__name__)
 
 class Step(Object):
     """ Basic execution unit.
-    It should defines its Input, Output
+    It should define its Input, Output
     and define a run method.
     """
 
-    def __init__(self, **kwargs):
-        Object.__init__(self, **kwargs)
+    def __init__(self, interactive=False, needsGPU=True, **kwargs):
+        super().__init__()
         self._prerequisites = CsvList()  # which steps needs to be done first
         self.status = String()
         self.initTime = String()
         self.endTime = String()
         self._error = String()
-        self.interactive = Boolean(False)
+        self.interactive = Boolean(interactive)
         self._resultFiles = String()
+        self._needsGPU = Boolean(needsGPU)
         self._index = None
 
+    def needsGPU(self) -> bool:
+        return self._needsGPU.get()
     def getIndex(self):
         return self._index
 
@@ -214,7 +216,7 @@ class Step(Object):
                 self.status.set(status)
 
         except PyworkflowException as e:
-            print(pwutils.redStr(str(e)))
+            logger.info(pwutils.redStr(str(e)))
             self.setFailed(str(e))
         except Exception as e:
             self.setFailed(str(e))
@@ -230,7 +232,7 @@ class FunctionStep(Step):
     This class will ease the insertion of Protocol function steps
     through the function _insertFunctionStep"""
 
-    def __init__(self, func=None, funcName=None, *funcArgs, **kwargs):
+    def __init__(self, func=None, funcName=None, *funcArgs,  wait=False, interactive=False, needsGPU=True):
         """
          Params:
             func: the function that will be executed.
@@ -238,13 +240,12 @@ class FunctionStep(Step):
             *funcArgs: argument list passed to the function (serialized and stored)
             **kwargs: extra parameters.
         """
-        Step.__init__(self)
+        super().__init__(interactive=interactive, needsGPU=needsGPU)
         self._func = func  # Function should be set before run
         self._args = funcArgs
         self.funcName = String(funcName)
         self.argsStr = String(json.dumps(funcArgs, default=lambda x: None))
-        self.setInteractive(kwargs.get('interactive', False))
-        if kwargs.get('wait', False):
+        if wait:
             self.setStatus(STATUS_WAITING)
 
     def _runFunc(self):
@@ -280,7 +281,7 @@ class FunctionStep(Step):
         return not self.__eq__(other)
 
     def __str__(self):
-        return self.funcName.get()
+        return "%s - %s" % (self._objId ,self.funcName.get())
 
 
 class RunJobStep(FunctionStep):
@@ -328,7 +329,6 @@ class Protocol(Step):
     """
 
     # Version where protocol appeared first time
-    _lastUpdateVersion = pw.VERSION_1
     _stepsCheckSecs = pw.Config.getStepsCheckSeconds()
     # Protocol develop status: PROD, BETA, NEW
     _devStatus = pw.PROD
@@ -916,6 +916,27 @@ class Protocol(Step):
 
         return s
 
+    def getOutputSuffix(self, outputPrefix):
+        """ Return the suffix to be used for a new output.
+        For example: output3DCoordinates7.
+        It should take into account previous outputs
+        and number with a higher value.
+        """
+        maxCounter = -1
+        for attrName, _ in self.iterOutputAttributes():
+            suffix = attrName.replace(outputPrefix, '')
+            try:
+                counter = int(suffix)
+            except:
+                counter = 1  # when there is not number assume 1
+            maxCounter = max(counter, maxCounter)
+
+        return str(maxCounter + 1) if maxCounter > 0 else ''  # empty if not output
+
+    def getNextOutputName(self, outputPrefix):
+        """Return the name to be used for a new output."""
+        return outputPrefix + self.getOutputSuffix(outputPrefix)
+
     def copyDefinitionAttributes(self, other):
         """ Copy definition attributes to other protocol. """
         for paramName, _ in self.iterDefinitionAttributes():
@@ -996,12 +1017,11 @@ class Protocol(Step):
         """
         pass
 
-    def __insertStep(self, step, **kwargs):
+    def __insertStep(self, step, prerequisites=None):
         """ Insert a new step in the list.
 
         :param prerequisites: a single integer or a list with the steps index that need to be done
                            previous to the current one."""
-        prerequisites = kwargs.get('prerequisites', None)
 
         if prerequisites is None:
             if len(self._steps):
@@ -1097,7 +1117,7 @@ class Protocol(Step):
         """
         return self._getPath(os.path.basename(path))
 
-    def _insertFunctionStep(self, func, *funcArgs, **kwargs):
+    def _insertFunctionStep(self, func, *funcArgs, prerequisites=None, wait=False, interactive=False, needsGPU=True):
         """
          Params:
            func: the function itself or, optionally, the name (string) of the function to be run in the Step.
@@ -1115,24 +1135,24 @@ class Protocol(Step):
         if not callable(func):
             raise Exception("Protocol._insertFunctionStep: '%s' is not callable"
                             % func)
-        step = FunctionStep(func, func.__name__, *funcArgs, **kwargs)
-
-        return self.__insertStep(step, **kwargs)
-
-    def _insertRunJobStep(self, progName, progArguments, resultFiles=[],
-                          **kwargs):
-        """ Insert an Step that will simple call runJob function
-        **args: see __insertStep
-        """
-        return self._insertFunctionStep('runJob', progName, progArguments,
-                                        **kwargs)
-
-    def _insertCopyFileStep(self, sourceFile, targetFile, **kwargs):
-        """ Shortcut function to insert an step for copying a file to a destiny. """
-        step = FunctionStep(pwutils.copyFile, 'copyFile', sourceFile,
-                            targetFile,
-                            **kwargs)
-        return self.__insertStep(step, **kwargs)
+        step = FunctionStep(func, func.__name__, *funcArgs, wait=wait, interactive=interactive, needsGPU=needsGPU)
+
+        return self.__insertStep(step,prerequisites)
+
+    # def _insertRunJobStep(self, progName, progArguments, resultFiles=[],
+    #                       **kwargs):
+    #     """ Insert an Step that will simple call runJob function
+    #     **args: see __insertStep
+    #     """
+    #     return self._insertFunctionStep('runJob', progName, progArguments,
+    #                                     **kwargs)
+    #
+    # def _insertCopyFileStep(self, sourceFile, targetFile, **kwargs):
+    #     """ Shortcut function to insert a step for copying a file to a destiny. """
+    #     step = FunctionStep(pwutils.copyFile, 'copyFile', sourceFile,
+    #                         targetFile,
+    #                         **kwargs)
+    #     return self.__insertStep(step, **kwargs)
 
     def _enterDir(self, path):
         """ Enter into a new directory path and store the current path.
@@ -1196,7 +1216,7 @@ class Protocol(Step):
         protocol that allow some kind of continue (such as ctf estimation).
         """
         for step in self.loadSteps():
-            self.__insertStep(step)
+            self.__insertStep(step, )
 
     def __findStartingStep(self):
         """ From a previous run, compare self._steps and self._prevSteps
@@ -2518,7 +2538,7 @@ class ProtStreamingBase(Protocol):
         self.stepsExecutionMode = STEPS_PARALLEL
     def _insertAllSteps(self):
         # Insert the step that generates the steps
-        self._insertFunctionStep(self.resumableStepGeneratorStep, str(datetime.now()))
+        self._insertFunctionStep(self.resumableStepGeneratorStep, str(datetime.now()), needsGPU=False)
 
     def resumableStepGeneratorStep(self, ts):
         """ This allow to resume protocols. ts is the time stamp so this stap is alway different form previous exceution"""

pyworkflow/template.py

@@ -10,7 +10,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 class Template:
-    def __init__(self, source, name, description):
+    def __init__(self, source, name, description=""):
         self.source = source
         # Tidy up templates names: removing .json.template and .json (when passed as parameter)
         self.name = name

pyworkflow/utils/graph.py

@@ -26,6 +26,8 @@
 """
 This module define a Graph class and some utilities 
 """
+import logging
+logger = logging.getLogger(__name__)
 
 
 class Node(object):
@@ -33,7 +35,7 @@ class Node(object):
     _count = 1
 
     def __init__(self, name=None, label=None):
-        self._childs = []
+        self._children = []
         self._parents = []
 
         if name is None:
@@ -44,6 +46,7 @@ class Node(object):
         if label is None:
             label = name
         self._label = label
+        self._fixed = False
 
     def getName(self):
         return self._name
@@ -57,13 +60,13 @@ class Node(object):
     def isRoot(self):
         return len(self._parents) == 0
 
-    def getChilds(self):
-        return self._childs
+    def getChildren(self):
+        return self._children
 
     def addChild(self, *nodes):
         for n in nodes:
-            if n not in self._childs:
-                self._childs.append(n)
+            if n not in self._children:
+                self._children.append(n)
                 n._parents.append(self)
 
     def getParent(self):
@@ -78,36 +81,36 @@ class Node(object):
     def getParents(self):
         return self._parents
 
-    def iterChilds(self):
-        """ Iterate over all childs and subchilds. 
-        Nodes can be visited more than once if have
+    def iterChildren(self):
+        """ Iterate over all children and sub-children.
+        Nodes can be visited more than once if it has
         more than one parent.
         """
-        for child in self._childs:
-            for c in child.iterChilds():
+        for child in self._children:
+            for c in child.iterChildren():
                 yield c
 
         yield self
 
-    def countChilds(self, visitedNode=None, count=0):
+    def countChildren(self, visitedNode=None, count=0):
         """ Iterate over all childs and subchilds.
         Nodes can be visited once
         """
-        for child in self._childs:
+        for child in self._children:
             if child._name not in visitedNode:
                 visitedNode[child._name] = True
-            child.countChilds(visitedNode)
+            child.countChildren(visitedNode)
         return len(visitedNode)
 
 
-    def iterChildsBreadth(self):
+    def iterChildrenBreadth(self):
         """ Iter child nodes in a breadth-first order
         """
-        for child in self._childs:
+        for child in self._children:
             yield child
 
-        for child in self._childs:
-            for child2 in child.iterChildsBreadth():
+        for child in self._children:
+            for child2 in child.iterChildrenBreadth():
                 yield child2
 
     def __str__(self):
@@ -133,13 +136,13 @@ class Graph(object):
     def _registerNode(self, node):
         self._nodes.append(node)
         self._nodesDict[node.getName()] = node
-        for child in node.getChilds():
+        for child in node.getChildren():
             self._registerNode(child)
 
-    def getRoot(self):
+    def getRoot(self) -> Node:
         return self._root
 
-    def createNode(self, nodeName, nodeLabel=None):
+    def createNode(self, nodeName, nodeLabel=None) -> Node:
         """ Add a node to the graph """
         node = Node(nodeName, nodeLabel)
         self._registerNode(node)
@@ -150,7 +153,7 @@ class Graph(object):
         """ Register an alias name for the node. """
         self._nodesDict[aliasName] = node
 
-    def getNode(self, nodeName):
+    def getNode(self, nodeName) -> Node:
         return self._nodesDict.get(nodeName, None)
 
     def getNodeNames(self):
@@ -164,33 +167,3 @@ class Graph(object):
         """ Return all nodes that have no parent. """
         return [n for n in self._nodes if n.isRoot()]
 
-    def printNodes(self):
-        for node in self.getNodes():
-            print("Node: ", node)
-            print(" Childs: ", ','.join([c.getLabel()
-                                         for c in node.getChilds()]))
-
-    def _escape(self, label):
-        return label.replace('.', '_').replace(' ', '_').replace('-', '_').replace('___', '_')
-
-    def printDot(self, useId=True):
-        """ If useId is True, use the node id for label the graph.
-         If not, use the run name.
-        """
-
-        def getLabel(node):
-            if useId:
-                return node.getName()
-            else:
-                return node.getLabel()
-
-        dotStr = "\ndigraph {\n"
-
-        for node in self.getNodes():
-            for child in node.getChilds():
-                nodeLabel = self._escape(getLabel(node))
-                childLabel = self._escape(getLabel(child))
-                dotStr += "   %s -> %s;\n" % (nodeLabel, childLabel)
-        dotStr += "}"
-        print(dotStr)
-        return dotStr

pyworkflow/utils/properties.py

@@ -29,7 +29,7 @@
 This module defines the text used in the application.
 """
 # NOTE: DO NOT REMOVE UNTIL plugin manager uses Config.SCIPION_MAIN_COLOR and is released
-from pyworkflow.constants import Color
+from pyworkflow.constants import Color, DOCSITEURLS
 from PIL import Image
 
 class Message:
@@ -124,6 +124,7 @@ class Message:
     LABEL_PARALLEL = 'Parallel'
     LABEL_HOST = 'Host'
     LABEL_THREADS = 'Threads'
+    LABEL_SCIPION_THREADS = 'Scipion threads'
     LABEL_MPI = 'MPI'
     LABEL_QUEUE = 'Use a queue engine?'
 
@@ -140,18 +141,21 @@ Each step could be computationally intensive, that's why
 the *Continue* execution mode will try to continue from the
 last completed step. On the other hand, the *Restart* mode
 will clean the whole run directory and start from scratch.    
-    """    
-
-    HELP_MPI_THREADS = """
-Define the number of processors to be used in the execution.
-*MPI*: This is a number of independent processes
-       that communicate through message passing
-       over the network (or the same computer).
-*Threads*: This refers to different execution threads 
-       in the same process that can share memory. They run in
-       the same computer.     
     """
 
+    HELP_PARALLEL_HEADER = 'Define the number of processors to be used in the execution.\nCheck %s for more detailed info.\n\n' % DOCSITEURLS.THREADS_MPIS_AND_GPUS
+    HELP_PARALLEL_MPI = ("*MPI*:\nThis is a number of independent processes"
+                         " that communicate through message passing "
+                         "over the network (or the same computer).\n")
+    HELP_PARALLEL_THREADS = ("*Threads*:\nThis refers to different execution threads in the same process that "
+                             "can share memory. They run in the same computer. This value is an argument"
+                             " passed to the program integrated")
+
+    HELP_SCIPION_THREADS = ("*Scipion threads*:\n threads created by Scipion to run the steps."
+                            " 1 thread is always used by the master/main process. Then extra threads will allow"
+                            " this protocol to run several steps at the same time, taking always into account "
+                            "restrictions to previous steps and 'theoretical GPU availability'")
+
     HELP_USEQUEUE = """
     Click Yes if you want to send this execution to a queue engine like Slurm, Torque, ...
     The queue commands to launch and stop jobs should be configured at
@@ -446,7 +450,7 @@ class Sprite:
     @classmethod
     def getSpritesFile(cls):
         from pyworkflow import Config
-        return Config.SCIPION_SPRITES_FILE
+        return Config.getSpritesFile()
     @classmethod
     def loadSprites(cls):
         """ Loads the image of the sprite"""

pyworkflowtests/protocols.py

@@ -55,16 +55,16 @@ class SleepingProtocol(pwprot.Protocol):
     def _insertAllSteps(self):
         print("Inserting all steps...")
         for i in range(self.numberOfSleeps.get()):
-            self._insertFunctionStep('sleepStep', i + 1)
+            self._insertFunctionStep(self.sleepStep, i + 1)
 
 
 class ParallelSleepingProtocol(SleepingProtocol):
     def _insertAllSteps(self):
-        step1 = self._insertFunctionStep('sleepStep', 1)
+        step1 = self._insertFunctionStep(self.sleepStep, 1)
         n = 2
         deps = [step1]
         for i in range(n):
-            self._insertFunctionStep('sleepStep')
+            self._insertFunctionStep(self.sleepStep)
 
 class ConcurrencyProtocol(SleepingProtocol):
     """ Protocol to test concurrency access to sets"""

pyworkflowtests/tests/test_protocol_execution.py

@@ -22,11 +22,13 @@
 # *  e-mail address 'scipion@cnb.csic.es'
 # *
 # **************************************************************************
+import threading
 
 import pyworkflow.tests as pwtests
 import pyworkflow.mapper as pwmapper
 import pyworkflow.protocol as pwprot
 from pyworkflow.project import Project
+from pyworkflow.protocol.constants import VOID_GPU
 
 
 # TODO: this test seems not to be finished.
@@ -70,3 +72,64 @@ class TestProtocolExecution(pwtests.BaseTest):
         prot2 = mapper2.selectById(prot.getObjId())
         
         self.assertEqual(prot.endTime.get(), prot2.endTime.get())
+
+    def test_gpuSlots(self):
+        """ Test gpu slots are properly composed in combination of threads"""
+
+        # Test basic GPU setu methods
+        stepExecutor = pwprot.ThreadStepExecutor(None, 1, gpuList=None)
+
+
+        self.assertEqual(stepExecutor.cleanVoidGPUs([0,1]), [0,1],
+                         "CleanVoidGpus does not work in absence of void GPUS")
+
+        self.assertEqual(stepExecutor.cleanVoidGPUs([0, VOID_GPU]), [0],
+                         "CleanVoidGpus does not work with a void GPU")
+
+        self.assertEqual(stepExecutor.cleanVoidGPUs([VOID_GPU, VOID_GPU]), [],
+                         "CleanVoidGpus does not work with all void GPU")
+
+
+        currThread = threading.currentThread()
+        currThread.thId = 1
+        self.assertEqual(stepExecutor.getGpuList(),[], "Gpu list should be empty")
+
+        # 2 threads 1 GPU
+        stepExecutor = pwprot.ThreadStepExecutor(None, 2, gpuList=[1])
+        self.assertEqual(stepExecutor.getGpuList(),[1], "Gpu list should be [1]")
+
+        currThread.thId = 2
+        self.assertEqual(stepExecutor.getGpuList(),[], "Gpu list should be empty after a second request")
+
+
+        # 2 threads 3 GPUs
+        stepExecutor = pwprot.ThreadStepExecutor(None, 2, gpuList=[0,1,2])
+        self.assertEqual(stepExecutor.getGpuList(),[0,1], "Gpu list should be [0,1]")
+
+        currThread.thId = 1
+        self.assertEqual(stepExecutor.getGpuList(),[2], "Gpu list should be [2] after a second request")
+
+
+        # 2 threads 4 GPUs with void gpus
+        stepExecutor = pwprot.ThreadStepExecutor(None, 2, gpuList=[0,1,2, VOID_GPU])
+        self.assertEqual(stepExecutor.getGpuList(),[0,1], "Gpu list should be [0,1]")
+
+        currThread.thId = 2
+        self.assertEqual(stepExecutor.getGpuList(),[2], "Gpu list should be [2] after a second request without the void gpu")
+
+        # less GPUs than threads. No extension should happen
+        stepExecutor = pwprot.ThreadStepExecutor(None, 4, gpuList=[0, VOID_GPU, 2])
+        self.assertEqual(stepExecutor.getGpuList(), [0], "Gpu list should not be extended")
+
+        currThread.thId = 1
+        self.assertEqual(stepExecutor.getGpuList(), [2],
+                         "Gpu list should be [2] after a second request, skipping the VOID gpu")
+
+        currThread.thId = 3
+        self.assertEqual(stepExecutor.getGpuList(), [], "Gpu list should be empty ather all GPU slots are busy")
+
+
+
+
+
+

{scipion_pyworkflow-3.7.0.dist-info → scipion_pyworkflow-3.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scipion-pyworkflow
-Version: 3.7.0
+Version: 3.9.0
 Summary: Simple workflow platform used in scientific applications, initially developed within the Scipion framework for image processing in Electron Microscopy.
 Home-page: https://github.com/scipion-em/scipion-pyworkflow
 Author: J.M. De la Rosa Trevin, Roberto Marabini, Grigory Sharov, Josue Gomez Blanco, Pablo Conesa, Yunior Fonseca Reyna
@@ -18,18 +18,18 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering
 License-File: LICENSE.txt
-Requires-Dist: bibtexparser <=1.4.1
-Requires-Dist: psutil <=5.9.6
+Requires-Dist: bibtexparser<=1.4.1
+Requires-Dist: psutil<=5.9.6
 Requires-Dist: tkcolorpicker
-Requires-Dist: distro <=1.8
-Requires-Dist: importlib-metadata <=6.8.0
-Requires-Dist: matplotlib ==3.7.3 ; python_version == "3.8"
-Requires-Dist: numpy ==1.24.4 ; python_version == "3.8"
-Requires-Dist: configparser ==6.0.0 ; python_version >= "3.8"
-Requires-Dist: pillow ==10.1.0 ; python_version >= "3.8"
-Requires-Dist: requests ==2.31.0 ; python_version >= "3.8"
-Requires-Dist: matplotlib ==3.8.1 ; python_version >= "3.9"
-Requires-Dist: numpy ==1.26.1 ; python_version >= "3.9"
+Requires-Dist: distro<=1.8
+Requires-Dist: importlib-metadata<=6.8.0
+Requires-Dist: matplotlib==3.7.3; python_version == "3.8"
+Requires-Dist: numpy==1.24.4; python_version == "3.8"
+Requires-Dist: configparser==6.0.0; python_version >= "3.8"
+Requires-Dist: pillow==10.1.0; python_version >= "3.8"
+Requires-Dist: requests==2.31.0; python_version >= "3.8"
+Requires-Dist: matplotlib==3.8.1; python_version >= "3.9"
+Requires-Dist: numpy==1.26.1; python_version >= "3.9"
 
 .. image:: https://img.shields.io/pypi/v/scipion-pyworkflow.svg
         :target: https://pypi.python.org/pypi/scipion-pyworkflow