scipion-pyworkflow 3.10.1__tar.gz → 3.10.3__tar.gz

{scipion_pyworkflow-3.10.1 → scipion_pyworkflow-3.10.3}/CHANGES.txt

@@ -1,3 +1,5 @@
+V3.10.3: hotfix: Parallelization with GPUs is fixed and step queue executor fixed with the new implementation.
+V3.10.2: hotfix: Executor frees the GPU/s used via step id instead of node.
 V3.10.1: hotfix: Avoid double Gpu assignment in scipion parallelize protocols with concurrent GPU steps
 V3.10.0
 developers:

{scipion_pyworkflow-3.10.1/scipion_pyworkflow.egg-info → scipion_pyworkflow-3.10.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scipion-pyworkflow
-Version: 3.10.1
+Version: 3.10.3
 Summary: Simple workflow platform used in scientific applications, initially developed within the Scipion framework for image processing in Electron Microscopy.
 Home-page: https://github.com/scipion-em/scipion-pyworkflow
 Author: J.M. De la Rosa Trevin, Roberto Marabini, Grigory Sharov, Josue Gomez Blanco, Pablo Conesa, Yunior Fonseca Reyna

{scipion_pyworkflow-3.10.1 → scipion_pyworkflow-3.10.3}/pyworkflow/constants.py

@@ -43,7 +43,7 @@ VERSION_1 = '1.0.0'
 VERSION_1_1 = '1.1.0'
 VERSION_1_2 = '1.2.0'
 VERSION_2_0 = '2.0.0'
-VERSION_3_0 = '3.10.1'
+VERSION_3_0 = '3.10.3'
 
 # For a new release, define a new constant and assign it to LAST_VERSION
 # The existing one has to be added to OLD_VERSIONS list.

{scipion_pyworkflow-3.10.1 → scipion_pyworkflow-3.10.3}/pyworkflow/protocol/executor.py

@@ -287,7 +287,7 @@ class ThreadStepExecutor(StepExecutor):
         # Some nodes/threads do not use gpus so may not be booked and not in the dictionary
         if gpus is not None:
             self.gpuDict.pop(node)
-            self.gpuDict[-node-1] = gpus
+            self.gpuDict[-node] = gpus
             logger.info("GPUs %s freed from step %s" % (gpus, node))
         else:
             logger.debug("step id %s not found in GPU slots" % node)
@@ -337,7 +337,7 @@ class ThreadStepExecutor(StepExecutor):
             for node in nodesFinished:
                 step = runningSteps.pop(node)  # remove entry from runningSteps
                 freeNodes.append(node)  # the node is available now
-                self.freeGpusSlot(node)
+                self.freeGpusSlot(step.getObjId())
                 # Notify steps termination and check if we should continue
                 doContinue = stepFinishedCallback(step)
                 if not doContinue:
@@ -392,8 +392,6 @@ class QueueStepExecutor(ThreadStepExecutor):
         self.submitDict = submitDict
         # Command counter per thread
         self.threadCommands = {}
-        for threadId in range(nThreads):
-            self.threadCommands[threadId] = 0
 
         if nThreads > 1:
             self.runJobs = ThreadStepExecutor.runSteps
@@ -411,6 +409,15 @@ class QueueStepExecutor(ThreadStepExecutor):
 
         logger.debug("Updated gpus ids rebase starting from 0: %s per thread" %self.gpuDict)
 
+    def getThreadJobId(self, stepId):
+        """ Returns the job id extension assigned to each thread/step """
+        if not stepId in self.threadCommands:
+            self.threadCommands[stepId] = 0
+
+        self.threadCommands[stepId] += 1
+
+        return self.threadCommands[stepId]
+
     def runJob(self, log, programName, params, numberOfMpi=1, numberOfThreads=1, env=None, cwd=None, executable=None):
         threadId = threading.current_thread().thId
         submitDict = dict(self.hostConfig.getQueuesDefault())
@@ -418,8 +425,8 @@ class QueueStepExecutor(ThreadStepExecutor):
         submitDict['JOB_COMMAND'] = process.buildRunCommand(programName, params, numberOfMpi,
                                                             self.hostConfig, env,
                                                             gpuList=self.getGpuList())
-        self.threadCommands[threadId] += 1
-        subthreadId = '-%s-%s' % (threadId, self.threadCommands[threadId])
+        threadJobId = self.getThreadJobId(threadId)
+        subthreadId = '-%s-%s' % (threadId, threadJobId)
         submitDict['JOB_NAME'] = submitDict['JOB_NAME'] + subthreadId
         submitDict['JOB_SCRIPT'] = os.path.abspath(removeExt(submitDict['JOB_SCRIPT']) + subthreadId + ".job")
         submitDict['JOB_LOGS'] = os.path.join(getParentFolder(submitDict['JOB_SCRIPT']), submitDict['JOB_NAME'])

{scipion_pyworkflow-3.10.1 → scipion_pyworkflow-3.10.3}/pyworkflow/protocol/protocol.py

@@ -1934,7 +1934,9 @@ class Protocol(Step):
              'JOB_CORES': self.numberOfMpi.get() * self.numberOfThreads.get(),
              'JOB_HOURS': 72,
              'GPU_COUNT': len(self.getGpuList()),
-             'QUEUE_FOR_JOBS': 'N'
+             'QUEUE_FOR_JOBS': 'N',
+             'SCIPION_PROJECT': self.getProject().getShortName(),
+             'SCIPION_PROTOCOL': self.getRunName()
              }
         d.update(queueParams)
         return d

{scipion_pyworkflow-3.10.1 → scipion_pyworkflow-3.10.3/scipion_pyworkflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scipion-pyworkflow
-Version: 3.10.1
+Version: 3.10.3
 Summary: Simple workflow platform used in scientific applications, initially developed within the Scipion framework for image processing in Electron Microscopy.
 Home-page: https://github.com/scipion-em/scipion-pyworkflow
 Author: J.M. De la Rosa Trevin, Roberto Marabini, Grigory Sharov, Josue Gomez Blanco, Pablo Conesa, Yunior Fonseca Reyna