scipion-pyworkflow 3.9.2__tar.gz → 3.10.1__tar.gz

{scipion_pyworkflow-3.9.2 → scipion_pyworkflow-3.10.1}/CHANGES.txt

@@ -1,3 +1,8 @@
+V3.10.1: hotfix: Avoid double Gpu assignment in scipion parallelize protocols with concurrent GPU steps
+V3.10.0
+developers:
+  -  stepsExecutionMode = STEPS_SERIAL at class level. Should a class attribute for future versions. Now both approaches are valid.
+
 V3.9.2 - hotfix: rescue commented methods: Protocol._insertRunJobStep and Protocol._insertCopyFileStep
 V3.9.1 - hotfix: loading related objects (CTF - Mics in particle extraction)
 V3.9.0

{scipion_pyworkflow-3.9.2/scipion_pyworkflow.egg-info → scipion_pyworkflow-3.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scipion-pyworkflow
-Version: 3.9.2
+Version: 3.10.1
 Summary: Simple workflow platform used in scientific applications, initially developed within the Scipion framework for image processing in Electron Microscopy.
 Home-page: https://github.com/scipion-em/scipion-pyworkflow
 Author: J.M. De la Rosa Trevin, Roberto Marabini, Grigory Sharov, Josue Gomez Blanco, Pablo Conesa, Yunior Fonseca Reyna

{scipion_pyworkflow-3.9.2 → scipion_pyworkflow-3.10.1}/pyworkflow/constants.py

@@ -43,7 +43,7 @@ VERSION_1 = '1.0.0'
 VERSION_1_1 = '1.1.0'
 VERSION_1_2 = '1.2.0'
 VERSION_2_0 = '2.0.0'
-VERSION_3_0 = '3.9.2'
+VERSION_3_0 = '3.10.1'
 
 # For a new release, define a new constant and assign it to LAST_VERSION
 # The existing one has to be added to OLD_VERSIONS list.

{scipion_pyworkflow-3.9.2 → scipion_pyworkflow-3.10.1}/pyworkflow/protocol/executor.py

@@ -144,9 +144,9 @@ class StepExecutor:
 
 class StepThread(threading.Thread):
     """ Thread to run Steps in parallel. """
-    def __init__(self, thId, step, lock):
+    def __init__(self, step, lock):
         threading.Thread.__init__(self)
-        self.thId = thId
+        self.thId = step.getObjId()
         self.step = step
         self.lock = lock
 
@@ -262,7 +262,7 @@ class ThreadStepExecutor(StepExecutor):
                 return []
             else:
                 return gpus
-    def getFreeGpuSlot(self, nodeId=None):
+    def getFreeGpuSlot(self, stepId=None):
         """ Returns a free gpu slot available or None. If node is passed it also reserves it for that node
 
         :param node: node to make the reserve of Gpus
@@ -272,10 +272,10 @@ class ThreadStepExecutor(StepExecutor):
             if node < 0:
                 gpus = self.gpuDict[node]
 
-                if nodeId is not None:
+                if stepId is not None:
                     self.gpuDict.pop(node)
-                    self.gpuDict[nodeId] = gpus
-                    logger.info("GPUs %s assigned to thread %s" % (gpus, nodeId))
+                    self.gpuDict[stepId] = gpus
+                    logger.info("GPUs %s assigned to step %s" % (gpus, stepId))
                 else:
                     logger.info("Free gpu slot found at %s" % node)
                 return gpus
@@ -288,14 +288,14 @@ class ThreadStepExecutor(StepExecutor):
         if gpus is not None:
             self.gpuDict.pop(node)
             self.gpuDict[-node-1] = gpus
-            logger.info("GPUs %s freed from thread %s" % (gpus, node))
+            logger.info("GPUs %s freed from step %s" % (gpus, node))
         else:
-            logger.debug("node %s not found in GPU slots" % node)
+            logger.debug("step id %s not found in GPU slots" % node)
 
     def _isStepRunnable(self, step):
         """ Overwrite this method to check GPUs availability"""
 
-        if self.gpuList and step.needsGPU() and self.getFreeGpuSlot() is None:
+        if self.gpuList and step.needsGPU() and self.getFreeGpuSlot(step.getObjId()) is None:
             logger.info("Can't run step %s. Needs gpus and there are no free gpu slots" % step)
             return False
 
@@ -361,7 +361,7 @@ class ThreadStepExecutor(StepExecutor):
                         node = freeNodes.pop(0)  # take an available node
                         runningSteps[node] = step
                         logger.debug("Running step %s on node %s" % (step, node))
-                        t = StepThread(node, step, sharedLock)
+                        t = StepThread(step, sharedLock)
                         # won't keep process up if main thread ends
                         t.daemon = True
                         t.start()

{scipion_pyworkflow-3.9.2 → scipion_pyworkflow-3.10.1}/pyworkflow/protocol/protocol.py

@@ -358,6 +358,22 @@ class Protocol(Step):
     _package = None
     _plugin = None
 
+    # Maybe this property can be inferred from the
+    # prerequisites of steps, but is easier to keep it
+    stepsExecutionMode = STEPS_SERIAL
+
+    @classmethod
+    def modeSerial(cls):
+        """ Returns true if steps are run one after another"""
+        # Maybe this property can be inferred from the
+        # prerequisites of steps, but is easier to keep it
+        return cls.stepsExecutionMode == STEPS_SERIAL
+
+    @classmethod
+    def modeParallel(cls):
+        """ Returns true if steps are run in parallel"""
+        return not cls.modeSerial()
+
     def __init__(self, **kwargs):
         Step.__init__(self, **kwargs)
         self._size = None
@@ -410,9 +426,6 @@ class Protocol(Step):
         if not hasattr(self, 'hostFullName'):
             self.hostFullName = String()
 
-        # Maybe this property can be inferred from the
-        # prerequisites of steps, but is easier to keep it
-        self.stepsExecutionMode = STEPS_SERIAL
 
         # Run mode
         self.runMode = Integer(kwargs.get('runMode', MODE_RESUME))
@@ -1315,7 +1328,7 @@ class Protocol(Step):
                                         prot_id=self.getObjId(),
                                         prot_name=self.getClassName(),
                                         step_id=step._index))
-        if step.isFailed() and self.stepsExecutionMode == STEPS_PARALLEL:
+        if step.isFailed() and self.modeParallel():
             # In parallel mode the executor will exit to close
             # all working threads, so we need to close
             self._endRun()
@@ -2433,7 +2446,7 @@ def runProtocolMain(projectPath, protDbPath, protId):
     executor = None
     nThreads = max(protocol.numberOfThreads.get(), 1)
 
-    if protocol.stepsExecutionMode == STEPS_PARALLEL and nThreads > 1:
+    if protocol.modeParallel() and nThreads > 1:
         if protocol.useQueueForSteps():
             executor = QueueStepExecutor(hostConfig,
                                          protocol.getSubmitDict(),
@@ -2451,6 +2464,7 @@ def runProtocolMain(projectPath, protDbPath, protId):
         executor = StepExecutor(hostConfig,
                                 gpuList=protocol.getGpuList())
 
+    logger.info("Running protocol using the %s executor." % executor)
     protocol.setStepsExecutor(executor)
     # Finally run the protocol
     protocol.run()
@@ -2529,10 +2543,7 @@ class ProtStreamingBase(Protocol):
     Minimum number of threads is 3 and should run in parallel mode.
     """
 
-    def __init__(self, **kwargs):
-
-        super().__init__()
-        self.stepsExecutionMode = STEPS_PARALLEL
+    stepsExecutionMode = STEPS_PARALLEL
     def _insertAllSteps(self):
         # Insert the step that generates the steps
         self._insertFunctionStep(self.resumableStepGeneratorStep, str(datetime.now()), needsGPU=False)

{scipion_pyworkflow-3.9.2 → scipion_pyworkflow-3.10.1}/pyworkflow/utils/path.py

@@ -292,10 +292,11 @@ def createAbsLink(source, dest):
     """ Creates a link to a given file path"""
     if os.path.islink(dest):
         os.remove(dest)
-        
+
     if os.path.exists(dest):
         raise Exception('Destination %s os.path.exists and is not a link' % dest)
-
+    
+    source = os.path.abspath(source)
     os.symlink(source, dest)
 
 

{scipion_pyworkflow-3.9.2 → scipion_pyworkflow-3.10.1}/pyworkflow/utils/utils.py

@@ -498,7 +498,9 @@ class LazyDict(object):
 def parseBibTex(bibtexStr):
     """ Parse a bibtex file and return a dictionary. """
 
-    return bibtexparser.loads(bibtexStr).entries_dict
+    return bibtexparser.loads(bibtexStr,
+                              parser=bibtexparser.bparser.BibTexParser(common_strings=True)
+                              ).entries_dict
 
 
 

{scipion_pyworkflow-3.9.2 → scipion_pyworkflow-3.10.1/scipion_pyworkflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: scipion-pyworkflow
-Version: 3.9.2
+Version: 3.10.1
 Summary: Simple workflow platform used in scientific applications, initially developed within the Scipion framework for image processing in Electron Microscopy.
 Home-page: https://github.com/scipion-em/scipion-pyworkflow
 Author: J.M. De la Rosa Trevin, Roberto Marabini, Grigory Sharov, Josue Gomez Blanco, Pablo Conesa, Yunior Fonseca Reyna