scipion-pyworkflow 3.7.0__py3-none-any.whl → 3.9.0__py3-none-any.whl

pyworkflow/project/config.py

@@ -40,7 +40,7 @@ class ProjectSettings(pwobj.Object):
     COLOR_MODE_LABELS = 1
     COLOR_MODE_AGE = 2
     COLOR_MODE_SIZE = 3
-    COLOR_MODES = (COLOR_MODE_STATUS, COLOR_MODE_LABELS, COLOR_MODE_AGE, COLOR_MODE_SIZE)
+    COLOR_MODES = (COLOR_MODE_STATUS, COLOR_MODE_LABELS, COLOR_MODE_AGE)  # This has poor performance many cases, COLOR_MODE_SIZE)
 
     def __init__(self, confs={}, **kwargs):
         super().__init__(**kwargs)
@@ -112,6 +112,9 @@ class ProjectSettings(pwobj.Object):
     def setColorMode(self, colorMode):
         """ Set the color mode to use when drawing the graph.
         """
+        # Skip LABELS color mode to avoid double iteration
+        if colorMode == self.COLOR_MODE_LABELS:
+            colorMode+=1
         self.colorMode.set(colorMode)
 
     def statusColorMode(self):

pyworkflow/project/manager.py

@@ -97,9 +97,9 @@ class Manager(object):
         return projList
     
     def createProject(self, projectName, runsView=1, 
-                      hostsConf=None, protocolsConf=None, location=None):
+                      hostsConf=None, protocolsConf=None, location=None, comment=None):
         """Create a new project.
-        confs dict can contains customs .conf files 
+        confs dict can contain customs .conf files
         for: menus, protocols, or hosts
         """
         # Clean project name from undesired characters
@@ -118,7 +118,8 @@ class Manager(object):
         project = Project(pw.Config.getDomain(), projectPath)
         project.create(runsView=runsView, 
                        hostsConf=hostsConf, 
-                       protocolsConf=protocolsConf)
+                       protocolsConf=protocolsConf,
+                       comment=comment)
         # If location is not the default one create a symlink on self.PROJECTS directory
         if projectPath != self.getProjectPath(projectName):
             # JMRT: Let's create the link to the absolute path, since relative

pyworkflow/project/project.py

@@ -106,9 +106,11 @@ class Project(object):
         self.settings:config.ProjectSettings = None
         # Host configuration
         self._hosts = None
+
         #  Creation time should be stored in project.sqlite when the project
         # is created and then loaded with other properties from the database
         self._creationTime = None
+
         # Time stamp with the last run has been updated
         self._lastRunTime = None
 
@@ -145,7 +147,16 @@ class Project(object):
         """ Return the time when the project was created. """
         # In project.create method, the first object inserted
         # in the mapper should be the creation time
-        return self._creationTime
+        return self._creationTime.datetime()
+
+
+    def getComment(self):
+        """ Returns the project comment. Stored as CreationTime comment."""
+        return self._creationTime.getObjComment()
+
+    def setComment(self, newComment):
+        """ Sets the project comment """
+        self._creationTime.setObjComment(newComment)
 
     def getSettingsCreationTime(self):
         return self.settings.getCreationTime()
@@ -154,7 +165,7 @@ class Project(object):
         """ Returns the time elapsed from the creation to the last
         execution time. """
         if self._creationTime and self._lastRunTime:
-            creationTs = self._creationTime
+            creationTs = self.getCreationTime()
             lastRunTs = self._lastRunTime.datetime()
             return lastRunTs - creationTs
         return None
@@ -293,12 +304,16 @@ class Project(object):
         creationTime = self.mapper.selectBy(name=PROJECT_CREATION_TIME)
 
         if creationTime:  # CreationTime was found in project.sqlite
-            self._creationTime = creationTime[0].datetime()
+            ctStr = creationTime[0] # This is our String type instance
+
+            # We store it in mem as dateime
+            self._creationTime = ctStr
+
         else:
             # We should read the creation time from settings.sqlite and
             # update the CreationTime in the project.sqlite
-            self._creationTime = self.getSettingsCreationTime()
-            self._storeCreationTime(self._creationTime)
+            self._creationTime = pwobj.String(self.getSettingsCreationTime())
+            self._storeCreationTime()
 
     # ---- Helper functions to load different pieces of a project
     def _loadDb(self, dbPath):
@@ -360,7 +375,7 @@ class Project(object):
         return self.settings.getProtocolView()
 
     def create(self, runsView=1, readOnly=False, hostsConf=None,
-               protocolsConf=None):
+               protocolsConf=None, comment=None):
         """Prepare all required paths and files to create a new project.
 
         :param runsView: default view to associate the project with
@@ -376,7 +391,9 @@ class Project(object):
         # Create db through the mapper
         self.mapper = self.createMapper(self.dbPath)
         # Store creation time
-        self._storeCreationTime(dt.datetime.now())
+        self._creationTime = pwobj.String(dt.datetime.now())
+        self.setComment(comment)
+        self._storeCreationTime()
         # Load settings from .conf files and write .sqlite
         self.settings = self.createSettings(runsView=runsView,
                                             readOnly=readOnly)
@@ -386,12 +403,11 @@ class Project(object):
 
         self._loadHosts(hostsConf)
 
-    def _storeCreationTime(self, creationTime):
+    def _storeCreationTime(self, new=True):
         """ Store the creation time in the project db. """
         # Store creation time
-        creation = pwobj.String(objName=PROJECT_CREATION_TIME)
-        creation.set(creationTime)
-        self.mapper.insert(creation)
+        self._creationTime.setName(PROJECT_CREATION_TIME)
+        self.mapper.store(self._creationTime)
         self.mapper.commit()
 
     def _cleanData(self):
@@ -820,7 +836,7 @@ class Project(object):
         for prot in protocols:
             node = runsGraph.getNode(prot.strId())
             if node:
-                childs = [node.run for node in node.getChilds() if
+                childs = [node.run for node in node.getChildren() if
                           self.__validDependency(prot, node.run, protocols)]
                 if childs:
                     deps = [' ' + c.getRunName() for c in childs]
@@ -839,7 +855,7 @@ class Project(object):
         visitedNodes[int(node.getName())] = node
 
         def getDescendents(rootNode):
-            for child in rootNode.getChilds():
+            for child in rootNode.getChildren():
                 if int(child.getName()) not in visitedNodes:
                     visitedNodes[int(child.getName())] = child
                     getDescendents(child)
@@ -998,7 +1014,7 @@ class Project(object):
                 affectedProtocolsActive[protocol.getObjId()] = protocol
 
             node = runGraph.getNode(protocol.strId())
-            dependencies = [node.run for node in node.getChilds()]
+            dependencies = [node.run for node in node.getChildren()]
             for dep in dependencies:
                 if not dep.getObjId() in auxProtList:
                     auxProtList.append([dep.getObjId(), level])
@@ -1034,7 +1050,7 @@ class Project(object):
         node = self.getRunsGraph().getNode(protocol.strId())
         deps = []
 
-        for node in node.getChilds():
+        for node in node.getChildren():
             for _, inputObj in node.run.iterInputAttributes():
                 value = inputObj.get()
                 if (value is not None and
@@ -1187,7 +1203,7 @@ class Project(object):
                 node = g.getNode(prot.strId())
                 newProt = newDict[prot.getObjId()]
 
-                for childNode in node.getChilds():
+                for childNode in node.getChildren():
                     newChildProt = newDict.get(childNode.run.getObjId(), None)
 
                     if newChildProt:
@@ -1246,7 +1262,7 @@ class Project(object):
             protId = prot.getObjId()
             node = g.getNode(prot.strId())
 
-            for childNode in node.getChilds():
+            for childNode in node.getChildren():
                 childId = childNode.run.getObjId()
                 childProt = childNode.run
                 if childId in newDict:
@@ -1705,7 +1721,7 @@ class Project(object):
                         parentNode.addChild(node)
                         if os.environ.get('CHECK_CYCLIC_REDUNDANCY') and self._checkCyclicRedundancy(parentNode, node):
                             conflictiveNodes = set()
-                            for child in node.getChilds():
+                            for child in node.getChildren():
                                 if node in child._parents:
                                     child._parents.remove(node)
                                     conflictiveNodes.add(child)
@@ -1714,7 +1730,7 @@ class Project(object):
                                                       child.getLabel() + '(' + child.getName() + ')'))
 
                             for conflictNode in conflictiveNodes:
-                                node._childs.remove(conflictNode)
+                                node._children.remove(conflictNode)
 
                             return False
                         return True
@@ -1747,7 +1763,7 @@ class Project(object):
         def depthFirstSearch(node):
             visitedNodes.add(node)
             recursionStack.add(node)
-            for child in node.getChilds():
+            for child in node.getChildren():
                 if child not in visitedNodes:
                     if depthFirstSearch(child):
                         return True

pyworkflow/project/scripts/create.py

@@ -31,24 +31,28 @@ import os
 from pyworkflow.project import Manager
 import pyworkflow.utils as pwutils
 
+EMPTY_ARG = "-"
+
 
 def usage(error):
+
     print("""
     ERROR: %s
 
-    Usage: scipion python -m pyworkflow.project.scripts.create NAME [WORKFLOW] [LOCATION]
+    Usage: scipion python -m pyworkflow.project.scripts.create NAME [WORKFLOW] [LOCATION] [COMMENT]
         NAME: project name
         WORKFLOW: path to a Scipion json workflow
         LOCATION: where to create it, defaults to scipion default location
+        COMMENT: project comment, location is mandatory in this case... for a NULL LOCATION pass %s
 
         This script will create a project project, optionally based on a workflow file
-    """ % error)
+    """ % (error, EMPTY_ARG))
     sys.exit(1)
 
 
 n = len(sys.argv)
 
-if n < 2 or n > 4:
+if n < 2 or n > 5:
     usage("Incorrect number of input parameters")
 
 projName = sys.argv[1]
@@ -56,6 +60,12 @@ projName = sys.argv[1]
 jsonFile = None if n < 3 else os.path.abspath(sys.argv[2])
 location = None if n < 4 else sys.argv[3]
 
+# Location with - is None
+if location == EMPTY_ARG:
+    location = None
+
+comment = None if n < 5 else sys.argv[4]
+
 # This might not be working anymore for python3.
 # I'm getting invalid ELF header triggered by matplotlib -->from . import _tkagg
 # path = pw.join('gui', 'no-tkinter')
@@ -71,7 +81,7 @@ if manager.hasProject(projName):
 if jsonFile is not None and not os.path.exists(jsonFile):
     usage("Nonexistent json file: %s" % pwutils.red(jsonFile))
 
-project = manager.createProject(projName, location=location)
+project = manager.createProject(projName, location=location, comment=comment)
 
 if jsonFile is not None:
     protDict = project.loadProtocols(jsonFile)

pyworkflow/project/scripts/schedule.py

@@ -92,7 +92,7 @@ roots = runGraph.getRootNodes()
 # and the graph is lineal
 
 for root in roots:
-    for child in root.getChilds():
+    for child in root.getChildren():
         workflow, _ = project._getSubworkflow(child.run)
         for prot, level in workflow.values():
             protLabelName = prot.getObjLabel()

pyworkflow/protocol/constants.py

@@ -75,6 +75,7 @@ LEVEL_CHOICES = ('Normal', 'Advanced')
 # Param names for GPU processing
 USE_GPU = 'useGpu'
 GPU_LIST = 'gpuList'
+VOID_GPU = 99
 
 # Job management
 UNKNOWN_JOBID = -1

pyworkflow/protocol/executor.py

@@ -69,7 +69,7 @@ class StepExecutor:
                        numberOfMpi, numberOfThreads, 
                        self.hostConfig,
                        env=env, cwd=cwd, gpuList=self.getGpuList(), executable=executable)
-        
+
     def _getRunnable(self, steps, n=1):
         """ Return the n steps that are 'new' and all its
         dependencies have been finished, or None if none ready.
@@ -79,11 +79,16 @@ class StepExecutor:
         for s in steps:
             if (s.getStatus() == cts.STATUS_NEW and
                     all(steps[i-1].isFinished() for i in s._prerequisites)):
-                rs.append(s)
-                if len(rs) == n:
-                    break
+
+                if self._isStepRunnable(s):
+                    rs.append(s)
+                    if len(rs) == n:
+                        break
         return rs
-    
+    def _isStepRunnable(self, step):
+        """ Should be implemented by inherited classes to test extra conditions """
+        return True
+
     def _arePending(self, steps):
         """ Return True if there are pending steps (either running or waiting)
         that can be done and thus enable other steps to be executed.
@@ -169,28 +174,133 @@ class ThreadStepExecutor(StepExecutor):
         # all the threads
         self.gpuDict = {}
 
+        self._assignGPUperNode()
+
+    def _assignGPUperNode(self):
+        # If we have GPUs
         if self.gpuList:
-            nodes = range(nThreads)
+
+            nThreads = self.numberOfProcs
+
+            # Nodes: each concurrent steps
+            nodes = range(1, nThreads+1)
+
+            # Number of GPUs
             nGpu = len(self.gpuList)
 
+            # If more GPUs than threads
             if nGpu > nThreads:
-                chunk = int(nGpu / nThreads)
-                for i, node in enumerate(nodes):
-                    self.gpuDict[node] = list(self.gpuList[i*chunk:(i+1)*chunk])
+
+                # Get the ratio: 2 GPUs per thread? 3 GPUs per thread?
+                # 3 GPU and 2 threads is rounded to 1 (flooring)
+                step = int(nGpu / nThreads)
+                spare = nGpu % nThreads
+                fromPos = 0
+                # For each node(concurrent thread)
+                for node in nodes:
+                    # Store the GPUS per thread:
+                    # GPUs: 0 1 2
+                    # Threads 2 (step 1)
+                    # Node 0 : GPU 0 1
+                    # Node 1 : GPU 2
+
+                    extraGpu = 1 if spare>0 else 0
+                    toPos = fromPos + step +extraGpu
+                    gpusForNode = list(self.gpuList[fromPos:toPos])
+
+                    newGpusForNode = self.cleanVoidGPUs(gpusForNode)
+                    if len(newGpusForNode) == 0:
+                        logger.info("Gpu slot cancelled: all were null Gpus -> %s" % gpusForNode)
+                    else:
+                        logger.info("GPUs %s assigned to node %s" % (newGpusForNode, node))
+                        self.gpuDict[-node] = newGpusForNode
+
+                    fromPos = toPos
+                    spare-=1
+
             else:
                 # Expand gpuList repeating until reach nThreads items
                 if nThreads > nGpu:
-                    newList = self.gpuList * (int(nThreads/nGpu)+1)
-                    self.gpuList = newList[:nThreads]
-
-                for node, gpu in zip(nodes, self.gpuList):
-                    self.gpuDict[node] = [gpu]
+                    logger.warning("GPUs are no longer extended. If you want all GPUs to match threads repeat as many "
+                                   "GPUs as threads.")
+                    # newList = self.gpuList * (int(nThreads / nGpu) + 1)
+                    # self.gpuList = newList[:nThreads]
+
+                for index, gpu in enumerate(self.gpuList):
+
+                    if gpu == cts.VOID_GPU:
+                        logger.info("Void GPU (%s) found in the list. Skipping the slot." % cts.VOID_GPU)
+                    else:
+                        logger.info("GPU slot for gpu %s." % gpu)
+                        # Any negative number in the key means a free gpu slot. can't be 0!
+                        self.gpuDict[-index-1] = [gpu]
+
+    def cleanVoidGPUs(self, gpuList):
+        newGPUList=[]
+        for gpuid in gpuList:
+            if gpuid == cts.VOID_GPU:
+                logger.info("Void GPU detected in %s" % gpuList)
+            else:
+                newGPUList.append(gpuid)
+        return newGPUList
 
     def getGpuList(self):
         """ Return the GPU list assigned to current thread
         or empty list if not using GPUs. """
-        return self.gpuDict.get(threading.current_thread().thId, [])
-        
+
+        # If the node id has assigned gpus?
+        nodeId = threading.current_thread().thId
+        if nodeId in self.gpuDict:
+            gpus = self.gpuDict.get(nodeId)
+            logger.info("Reusing GPUs (%s) slot for %s" % (gpus, nodeId))
+            return gpus
+        else:
+
+            gpus = self.getFreeGpuSlot(nodeId)
+            if gpus is None:
+                logger.warning("Step on node %s is requesting GPUs but there isn't any available. Review configuration of threads/GPUs. Returning and empty list." % nodeId)
+                return []
+            else:
+                return gpus
+    def getFreeGpuSlot(self, nodeId=None):
+        """ Returns a free gpu slot available or None. If node is passed it also reserves it for that node
+
+        :param node: node to make the reserve of Gpus
+        """
+        for node in self.gpuDict.keys():
+            # This is a free node. Book it
+            if node < 0:
+                gpus = self.gpuDict[node]
+
+                if nodeId is not None:
+                    self.gpuDict.pop(node)
+                    self.gpuDict[nodeId] = gpus
+                    logger.info("GPUs %s assigned to thread %s" % (gpus, nodeId))
+                else:
+                    logger.info("Free gpu slot found at %s" % node)
+                return gpus
+
+        return None
+    def freeGpusSlot(self, node):
+        gpus = self.gpuDict.get(node, None)
+
+        # Some nodes/threads do not use gpus so may not be booked and not in the dictionary
+        if gpus is not None:
+            self.gpuDict.pop(node)
+            self.gpuDict[-node-1] = gpus
+            logger.info("GPUs %s freed from thread %s" % (gpus, node))
+        else:
+            logger.debug("node %s not found in GPU slots" % node)
+
+    def _isStepRunnable(self, step):
+        """ Overwrite this method to check GPUs availability"""
+
+        if self.gpuList and step.needsGPU() and self.getFreeGpuSlot() is None:
+            logger.info("Can't run step %s. Needs gpus and there are no free gpu slots" % step)
+            return False
+
+        return True
+
     def runSteps(self, steps, 
                  stepStartedCallback, 
                  stepFinishedCallback,
@@ -213,7 +323,9 @@ class ThreadStepExecutor(StepExecutor):
         sharedLock = threading.Lock()
 
         runningSteps = {}  # currently running step in each node ({node: step})
-        freeNodes = list(range(self.numberOfProcs))  # available nodes to send jobs
+        freeNodes = list(range(1, self.numberOfProcs+1))  # available nodes to send jobs
+        logger.info("Execution threads: %s" % freeNodes)
+        logger.info("Running steps using %s threads. 1 thread is used for this main proccess." % self.numberOfProcs)
 
         while True:
             # See which of the runningSteps are not really running anymore.
@@ -225,6 +337,7 @@ class ThreadStepExecutor(StepExecutor):
             for node in nodesFinished:
                 step = runningSteps.pop(node)  # remove entry from runningSteps
                 freeNodes.append(node)  # the node is available now
+                self.freeGpusSlot(node)
                 # Notify steps termination and check if we should continue
                 doContinue = stepFinishedCallback(step)
                 if not doContinue:
@@ -245,8 +358,9 @@ class ThreadStepExecutor(StepExecutor):
                         anyLaunched = True
                         step.setRunning()
                         stepStartedCallback(step)
-                        node = freeNodes.pop()  # take an available node
+                        node = freeNodes.pop(0)  # take an available node
                         runningSteps[node] = step
+                        logger.debug("Running step %s on node %s" % (step, node))
                         t = StepThread(node, step, sharedLock)
                         # won't keep process up if main thread ends
                         t.daemon = True
@@ -255,7 +369,7 @@ class ThreadStepExecutor(StepExecutor):
 
             if not anyLaunched:
                 if anyPending:  # nothing running
-                    time.sleep(0.5)
+                    time.sleep(3)
                 else:
                     break  # yeah, we are done, either failed or finished :)
 

pyworkflow/protocol/params.py

@@ -345,18 +345,15 @@ class Form(object):
         self.addParam('hostName', StringParam, default="localhost",
                       label='Execution host',
                       help='Select in which of the available do you want to launch this protocol.')
+
+        # NOTE: help messages for these parameters is defined at HELP_MPI_THREADS and used in form.py.
+
         if threads > 0:
             self.addParam('numberOfThreads', IntParam, default=threads,
-                          label='Threads',
-                          help='This option provides shared-memory parallelization on multi-core machines.'
-                                'It does not require any additional software.')
+                          label='Threads')
         if mpi > 0:
             self.addParam('numberOfMpi', IntParam, default=mpi,
-                          label='MPI processes',
-                          help='This option provides the number of independent processes spawned'
-                                'in parallel by <mpirun> command in a cluster, usually through'
-                                'a queue system. This will require that you have compile this software '
-                                'with <mpi> support.')
+                          label='MPI processes')
         if jobsize > 0:
             self.addParam('mpiJobSize', IntParam, default=jobsize,
                           label='MPI job size', condition="numberOfMpi>1",