scipion-pyworkflow 3.10.5__tar.gz → 3.10.6__tar.gz

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/CHANGES.txt

@@ -1,3 +1,17 @@
+V3.10.6:
+  - Improving subset creation
+  - PLUGIN_MODULE variable can be used in host.conf to customize mpirun command: mpirun.PLUGIN_MODULE --> mpirun.relion)
+       This approach implies having this pattern for all mpi plugins that use mpi.
+  - Alternatively, in the scipion.conf file a variable called PARALLEL_COMMAND_XXX could be defined for a more
+       versatile way to have a mpi command only for some plugins. Example:
+       PARALLEL_COMMAND_RELION=mpirun.relion -np %%(JOB_NODES)d --map-by node %%(COMMAND)s
+       NOTE: in this config file, %_ (valid and needed in host.conf) should be replaced by %%
+  - GPU Ids are automatically anonymized (0-len(GPUs)) when using queues
+  - Bug-fix: creation time is not loaded when loadig run.db (updating a protocol)
+  developers:
+  - Adding a Streaming section in ProtStreamingBase
+  - runJob call without %(GPU)s will not reserve GPU slots anymore.
+   
 V3.10.5: cancel usage of SCIPION_PROJECT in the host template
 V3.10.4: hotfix: Parallelization with GPUs is fixed and step queue executor fixed with the new implementation.
 V3.10.3: Failed hotfix
@@ -5,7 +19,7 @@ V3.10.2: hotfix: Executor frees the GPU/s used via step id instead of node.
 V3.10.1: hotfix: Avoid double Gpu assignment in scipion parallelize protocols with concurrent GPU steps
 V3.10.0
 developers:
-  -  stepsExecutionMode = STEPS_SERIAL at class level. Should a class attribute for future versions. Now both approaches are valid.
+  - stepsExecutionMode = STEPS_SERIAL at class level. Should be a class attribute for future versions. Now both approaches are valid.
 
 V3.9.2 - hotfix: rescue commented methods: Protocol._insertRunJobStep and Protocol._insertCopyFileStep
 V3.9.1 - hotfix: loading related objects (CTF - Mics in particle extraction)

{scipion_pyworkflow-3.10.5/scipion_pyworkflow.egg-info → scipion_pyworkflow-3.10.6}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: scipion-pyworkflow
-Version: 3.10.5
+Version: 3.10.6
 Summary: Simple workflow platform used in scientific applications, initially developed within the Scipion framework for image processing in Electron Microscopy.
 Home-page: https://github.com/scipion-em/scipion-pyworkflow
 Author: J.M. De la Rosa Trevin, Roberto Marabini, Grigory Sharov, Josue Gomez Blanco, Pablo Conesa, Yunior Fonseca Reyna
@@ -30,6 +30,16 @@ Requires-Dist: requests==2.31.0; python_version >= "3.8"
 Requires-Dist: tkcolorpicker
 Requires-Dist: distro<=1.8
 Requires-Dist: importlib-metadata<=6.8.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: summary
 
 .. image:: https://img.shields.io/pypi/v/scipion-pyworkflow.svg
         :target: https://pypi.python.org/pypi/scipion-pyworkflow

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/constants.py

@@ -43,7 +43,7 @@ VERSION_1 = '1.0.0'
 VERSION_1_1 = '1.1.0'
 VERSION_1_2 = '1.2.0'
 VERSION_2_0 = '2.0.0'
-VERSION_3_0 = '3.10.5'
+VERSION_3_0 = '3.10.6'
 
 # For a new release, define a new constant and assign it to LAST_VERSION
 # The existing one has to be added to OLD_VERSIONS list.
@@ -73,6 +73,16 @@ SCIPION_HOME_VAR = 'SCIPION_HOME'
 SCIPION_TESTS = 'SCIPION_TESTS'
 SCIPION_SCRATCH = 'SCIPION_SCRATCH'
 
+# VARIABLE names in host.conf
+PARALLEL_COMMAND_VAR = 'PARALLEL_COMMAND'
+PLUGIN_MODULE_VAR = 'PLUGIN_MODULE'
+QUEUE_FOR_JOBS = 'QUEUE_FOR_JOBS'
+
+
+# Launching constants
+RUN_JOB_GPU_PARAM = 'GPU' # Param name to "place" GPU ids used to build a run command.
+RUN_JOB_GPU_PARAM_SEARCH = "%("+ RUN_JOB_GPU_PARAM +")s"
+
 # FONT
 SCIPION_DEFAULT_FONT_SIZE = 10
 
@@ -215,4 +225,3 @@ DEFAULT_EXECUTION_ACTION_ALL = 3
 # Id field/attribute constants
 ID_COLUMN='id'
 ID_ATTRIBUTE='_objId'
-

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/gui/form.py

@@ -1743,7 +1743,7 @@ class FormWindow(Window):
                         entry = self._createBoundEntry(procFrame, pwutils.Message.VAR_MPI)
                         entry.grid(row=r2, column=c2 + 1, padx=(0, 5), sticky='w')
 
-                        helpMessage += pwutils.Message.HELP_PARALLEL_MPI
+                        helpMessage += '\n' + pwutils.Message.HELP_PARALLEL_MPI
 
 
                 btnHelp = IconButton(procFrame, pwutils.Message.TITLE_COMMENT,

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/gui/project/searchprotocol.py

@@ -139,7 +139,7 @@ class SearchProtocolWindow(SearchBaseWindow):
     def _addProtocolToTree(self, protList):
         """ Adds the items in protList to the tree
 
-        :param protList: List of tuples with all the values/colunms used in search ans shown in the tree"""
+        :param protList: List of tuples with all the values/columns used in the search search to show in the tree"""
 
         for key, label, installed, help, streamified, beta, new, updated, weight in protList:
             tag = ProtocolTreeConfig.getProtocolTag(installed == 'installed',

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/gui/project/viewprotocols_extra.py

@@ -167,7 +167,7 @@ class RunIOTreeProvider(pwgui.tree.TreeProvider):
         that is more readable for the user to pick the desired object.
         """
         label = 'None'
-        if obj:
+        if obj is not None:
             label = obj.getObjLabel()
             if not len(label.strip()):
                 parentLabel = parent.getObjLabel() if parent else 'None'
@@ -241,7 +241,6 @@ class RunIOTreeProvider(pwgui.tree.TreeProvider):
 
             return infoPtr
 
-
         if obj is None or not obj.hasValue():
             return None
 

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/mapper/sqlite.py

@@ -1027,13 +1027,16 @@ class SqliteFlatMapper(Mapper):
 
         return obj
         
-    def __iterObjectsFromRows(self, objRows, objectFilter=None):
+    def __iterObjectsFromRows(self, objRows, objectFilter=None, rowFilter=None):
         for objRow in objRows:
+            if rowFilter and not rowFilter(objRow):
+                continue
+
             obj = self.__objFromRow(objRow)
             if objectFilter is None or objectFilter(obj):
                 yield obj
         
-    def __objectsFromRows(self, objRows, iterate=False, objectFilter=None):
+    def __objectsFromRows(self, objRows, iterate=False, objectFilter=None, rowFilter=None):
         """Create a set of object from a set of rows
         Params:
             objRows: rows result from a db select.
@@ -1043,9 +1046,9 @@ class SqliteFlatMapper(Mapper):
         """
         if not iterate:
             return [obj.clone()
-                    for obj in self.__iterObjectsFromRows(objRows, objectFilter)]
+                    for obj in self.__iterObjectsFromRows(objRows, objectFilter, rowFilter)]
         else:
-            return self.__iterObjectsFromRows(objRows, objectFilter)
+            return self.__iterObjectsFromRows(objRows, objectFilter, rowFilter)
          
     def selectBy(self, iterate=False, objectFilter=None, **args):
         """Select object meetings some criteria"""
@@ -1053,7 +1056,7 @@ class SqliteFlatMapper(Mapper):
         return self.__objectsFromRows(objRows, iterate, objectFilter)
     
     def selectAll(self, iterate=True, objectFilter=None, orderBy=ID,
-                  direction='ASC', where='1', limit=None):
+                  direction='ASC', where='1', limit=None, rowFilter=None):
         # Just a sanity check for emtpy sets, that doesn't contains
         # 'Properties' table
         if not self.db.hasTable('Properties'):
@@ -1076,7 +1079,7 @@ and restarting scipion. Export command:
     export SQLITE_TMPDIR=. """ % str(e)
             raise OperationalError(msg)
         
-        return self.__objectsFromRows(objRows, iterate, objectFilter) 
+        return self.__objectsFromRows(objRows, iterate, objectFilter, rowFilter)
 
     def unique(self, labels, where=None):
         """ Returns a list (for a single label) or a dictionary with unique values for the passed labels.

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/object.py

@@ -1233,12 +1233,13 @@ class Set(Object):
         return self._getMapper().exists(itemId)
 
     def iterItems(self, orderBy='id', direction='ASC', where=None,
-                  limit=None, iterate=True):
+                  limit=None, iterate=True, rowFilter=None):
         return self._getMapper().selectAll(orderBy=orderBy,
                                            direction=direction,
                                            where=where,
                                            limit=limit,
-                                           iterate=iterate)  # has flat mapper, iterate is true
+                                           iterate=iterate,
+                                           rowFilter=rowFilter)  # has flat mapper, iterate is true
 
     def getFirstItem(self):
         """ Return the first item in the Set. """

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/plugin.py

@@ -508,7 +508,7 @@ class Domain:
 
         raiseMsg = "%s\n %s\n%s\n" % (msgStr, calling, hint)
         if doRaise:
-            raise Exception("\n\n" + raiseMsg)
+            raise ImportError("\n\n" + raiseMsg)
         else:
             logger.info(raiseMsg)
 

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/project/project.py

@@ -48,7 +48,7 @@ from pyworkflow.mapper import SqliteMapper
 from pyworkflow.protocol.constants import (MODE_RESTART, MODE_RESUME,
                                            STATUS_INTERACTIVE, ACTIVE_STATUS,
                                            UNKNOWN_JOBID, INITIAL_SLEEP_TIME, STATUS_FINISHED)
-from pyworkflow.protocol.protocol import ProtImportBase, Protocol
+from pyworkflow.protocol.protocol import Protocol
 
 from . import config
 
@@ -306,14 +306,17 @@ class Project(object):
         if creationTime:  # CreationTime was found in project.sqlite
             ctStr = creationTime[0] # This is our String type instance
 
-            # We store it in mem as dateime
+            # We store it in mem as datetime
             self._creationTime = ctStr
 
         else:
-            # We should read the creation time from settings.sqlite and
-            # update the CreationTime in the project.sqlite
-            self._creationTime = pwobj.String(self.getSettingsCreationTime())
-            self._storeCreationTime()
+
+            # If connected to project.sqlite and not any or the run.db
+            if self.path.endswith(PROJECT_DBNAME):
+                # We should read the creation time from settings.sqlite and
+                # update the CreationTime in the project.sqlite
+                self._creationTime = pwobj.String(self.getSettingsCreationTime())
+                self._storeCreationTime()
 
     # ---- Helper functions to load different pieces of a project
     def _loadDb(self, dbPath):
@@ -1961,37 +1964,31 @@ class Project(object):
         self.settings.setReadOnly(value)
 
     def fixLinks(self, searchDir):
-        logger.info("Fixing project links. Searching at %s" % searchDir)
+        logger.info(f"Fixing links for project {self.getShortName()}. Searching in: {searchDir}")
         runs = self.getRuns()
 
+        counter = 0
         for prot in runs:
-            print (prot)
-            broken = False
-            if isinstance(prot, ProtImportBase) or prot.getClassName() == "ProtImportMovies":
-                logger.info("Import detected")
-                for _, attr in prot.iterOutputAttributes():
-                    for f in attr.getFiles():
-                        if ':' in f:
-                            f = f.split(':')[0]
-
-                        if not os.path.exists(f):
-                            if not broken:
-                                broken = True
-                                logger.info("Found broken links in run: %s" %
-                                      pwutils.magenta(prot.getRunName()))
-                            logger.info("  Missing: %s" % pwutils.magenta(f))
-
-                            if os.path.islink(f):
-                                sourceFile = os.path.realpath(f)
-                                logger.info("    -> %s" % pwutils.red(sourceFile))
-
-                                newFile = pwutils.findFile(os.path.basename(sourceFile),
-                                                       searchDir,
-                                                       recursive=True)
-                                if newFile:
-                                    logger.info("  Found file %s, creating link... %s" % (newFile,
-                                        pwutils.green("   %s -> %s" % (f, newFile))))
-                                    pwutils.createAbsLink(newFile, f)
+            if prot.getClassName().startswith("ProtImport"):
+                runName = prot.getRunName()
+                logger.info(f"Found protocol {runName}")
+                for f in prot.getOutputFiles():
+                    if ':' in f:
+                        f = f.split(':')[0]
+
+                    if not os.path.exists(f):
+                        logger.info(f"\tMissing link: {f}")
+
+                        if os.path.islink(f):
+                            sourceFile = os.path.realpath(f)
+                            newFile = pwutils.findFileRecursive(os.path.basename(sourceFile),
+                                                                searchDir)
+                            if newFile:
+                                counter += 1
+                                logger.info(f"\t\tCreating link: {f} -> {newFile}")
+                                pwutils.createAbsLink(newFile, f)
+
+        logger.info(f"Fixed {counter} broken links")
 
     @staticmethod
     def cleanProjectName(projectName):

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/project/scripts/fix_links.py

@@ -3,12 +3,15 @@
 import sys
 import os
 import logging
-logging.basicConfig(level="INFO")
 
+from pyworkflow.config import Config
 from pyworkflow.project import Manager
 import pyworkflow.utils as pwutils
 
 
+logging.basicConfig(level=Config.SCIPION_LOG_LEVEL, format=Config.SCIPION_LOG_FORMAT)
+
+
 def usage(error):
     print("""
     ERROR: %s

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/protocol/constants.py

@@ -84,4 +84,4 @@ UNKNOWN_JOBID = -1
 SIZE_1KB = 1024
 SIZE_1MB = SIZE_1KB * SIZE_1KB
 SIZE_1GB = SIZE_1MB * SIZE_1KB
-SIZE_1TB = SIZE_1GB * SIZE_1KB
+SIZE_1TB = SIZE_1GB * SIZE_1KB

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/protocol/executor.py

@@ -39,6 +39,7 @@ import os
 
 import pyworkflow.utils.process as process
 from pyworkflow.utils.path import getParentFolder, removeExt
+from pyworkflow.constants import PLUGIN_MODULE_VAR, RUN_JOB_GPU_PARAM_SEARCH
 from . import constants as cts
 
 from .launch import _submit, UNKNOWN_JOBID, _checkJobStatus
@@ -59,16 +60,26 @@ class StepExecutor:
         """ Set protocol to append active jobs to its jobIds. """
         self.protocol = protocol
 
-    def runJob(self, log, programName, params,           
+    def getRunContext(self):
+        return {PLUGIN_MODULE_VAR: self.protocol.getPlugin().getName()}
+
+    def runJob(self, log, programName, params,
                numberOfMpi=1, numberOfThreads=1,
                env=None, cwd=None, executable=None):
         """ This function is a wrapper around runJob, 
         providing the host configuration. 
         """
         process.runJob(log, programName, params,
-                       numberOfMpi, numberOfThreads, 
+                       numberOfMpi, numberOfThreads,
                        self.hostConfig,
-                       env=env, cwd=cwd, gpuList=self.getGpuList(), executable=executable)
+                       env=env, cwd=cwd, gpuList=self._getGPUListForCommand(programName, params), executable=executable, context=self.protocol.getSubmitDict())
+
+    def _getGPUListForCommand(self, program, params):
+        """ Returns the list of GPUs if the program or the params have the GPU placeholder %(GPU)s """
+        if RUN_JOB_GPU_PARAM_SEARCH in params or RUN_JOB_GPU_PARAM_SEARCH in program:
+            return self.getGpuList()
+        else:
+            return []
 
     def _getRunnable(self, steps, n=1):
         """ Return the n steps that are 'new' and all its
@@ -258,7 +269,7 @@ class ThreadStepExecutor(StepExecutor):
 
             gpus = self.getFreeGpuSlot(nodeId)
             if gpus is None:
-                logger.warning("Step on node %s is requesting GPUs but there isn't any available. Review configuration of threads/GPUs. Returning and empty list." % nodeId)
+                logger.warning("Step on node %s is requesting GPUs but there isn't any available. Review configuration of threads/GPUs. Returning an empty list." % nodeId)
                 return []
             else:
                 return gpus
@@ -422,15 +433,20 @@ class QueueStepExecutor(ThreadStepExecutor):
         threadId = threading.current_thread().thId
         submitDict = dict(self.hostConfig.getQueuesDefault())
         submitDict.update(self.submitDict)
-        submitDict['JOB_COMMAND'] = process.buildRunCommand(programName, params, numberOfMpi,
-                                                            self.hostConfig, env,
-                                                            gpuList=self.getGpuList())
         threadJobId = self.getThreadJobId(threadId)
         subthreadId = '-%s-%s' % (threadId, threadJobId)
         submitDict['JOB_NAME'] = submitDict['JOB_NAME'] + subthreadId
         submitDict['JOB_SCRIPT'] = os.path.abspath(removeExt(submitDict['JOB_SCRIPT']) + subthreadId + ".job")
         submitDict['JOB_LOGS'] = os.path.join(getParentFolder(submitDict['JOB_SCRIPT']), submitDict['JOB_NAME'])
 
+        logger.debug("Variables available for replacement in submission command are: %s" % submitDict)
+
+        submitDict['JOB_COMMAND'] = process.buildRunCommand(programName, params, numberOfMpi,
+                                                            self.hostConfig, env,
+                                                            gpuList=self._getGPUListForCommand(programName, params),
+                                                            context=submitDict)
+
+
         jobid = _submit(self.hostConfig, submitDict, cwd, env)
         self.protocol.appendJobId(jobid)  # append active jobs
         self.protocol._store(self.protocol._jobId)

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/protocol/hosts.py

@@ -36,6 +36,7 @@ from configparser import RawConfigParser
 from collections import OrderedDict
 
 import pyworkflow as pw
+from pyworkflow import PARALLEL_COMMAND_VAR
 from pyworkflow.object import Object, String, Integer
 
 
@@ -199,7 +200,7 @@ class HostConfig(Object):
                 # Read the address of the remote hosts,
                 # using 'localhost' as default for backward compatibility
                 host.setAddress(get('ADDRESS', 'localhost'))
-                host.mpiCommand.set(get('PARALLEL_COMMAND'))
+                host.mpiCommand.set(get(PARALLEL_COMMAND_VAR))
                 host.queueSystem = QueueSystemConfig()
                 hostQueue = host.queueSystem  # shortcut
                 hostQueue.name.set(get('NAME'))

{scipion_pyworkflow-3.10.5 → scipion_pyworkflow-3.10.6}/pyworkflow/protocol/protocol.py

@@ -36,13 +36,15 @@ from pyworkflow.exceptions import ValidationException, PyworkflowException
 from pyworkflow.object import *
 import pyworkflow.utils as pwutils
 from pyworkflow.utils.log import getExtraLogInfo, STATUS, setDefaultLoggingContext
+from pyworkflow.constants import PLUGIN_MODULE_VAR, QUEUE_FOR_JOBS
 from .executor import StepExecutor, ThreadStepExecutor, QueueStepExecutor
 from .constants import *
-from .params import Form
+from .params import Form, IntParam
 from ..utils import getFileSize
 
 
 import  logging
+
 # Get the root logger
 logger = logging.getLogger(__name__)
 
@@ -67,6 +69,7 @@ class Step(Object):
 
     def needsGPU(self) -> bool:
         return self._needsGPU.get()
+
     def getIndex(self):
         return self._index
 
@@ -1117,10 +1120,7 @@ class Protocol(Step):
     def _getRelPathExecutionDir(self, *path):
         """ Return a relative path from the projdir. """
         # TODO  must be a bettis
-        return os.path.relpath(
-            self._getPath(*path),
-            os.path.dirname(os.path.dirname(self.workingDir.get()))
-        )
+        return os.path.relpath(self._getPath(*path), os.path.dirname(os.path.dirname(self.workingDir.get())))
 
     def _getBasePath(self, path):
         """ Take the basename of the path and get the path
@@ -1617,7 +1617,6 @@ class Protocol(Step):
                                         prot_id=self.getObjId(),
                                         prot_name=self.getClassName()))
 
-
     def getLogPaths(self):
         return [self.getStdoutLog(),self.getStderrLog() , self.getScheduleLog()]
 
@@ -1638,7 +1637,6 @@ class Protocol(Step):
         """ Return the steps.sqlite file under logs directory. """
         return self._getLogsPath('steps.sqlite')
 
-
     def _addChunk(self, txt, fmt=None):
         """
         Add text txt to self._buffer, with format fmt.
@@ -1921,21 +1919,26 @@ class Protocol(Step):
         queueName, queueParams = self.getQueueParams()
         hc = self.getHostConfig()
 
-        script = self._getLogsPath(hc.getSubmitPrefix() + self.strId() + '.job')
-        d = {'JOB_SCRIPT': script,
-             'JOB_LOGS': self._getLogsPath(hc.getSubmitPrefix() + self.strId()),
-             'JOB_NODEFILE': os.path.abspath(script.replace('.job', '.nodefile')),
-             'JOB_NAME': self.strId(),
+        d = {'JOB_NAME': self.strId(),
              'JOB_QUEUE': queueName,
              'JOB_NODES': self.numberOfMpi.get(),
              'JOB_THREADS': self.numberOfThreads.get(),
              'JOB_CORES': self.numberOfMpi.get() * self.numberOfThreads.get(),
              'JOB_HOURS': 72,
              'GPU_COUNT': len(self.getGpuList()),
-             'QUEUE_FOR_JOBS': 'N',
+             QUEUE_FOR_JOBS: 'N',
              'SCIPION_PROJECT': "SCIPION_PROJECT",  # self.getProject().getShortName(),
-             'SCIPION_PROTOCOL': self.getRunName()
+             'SCIPION_PROTOCOL': self.getRunName(),
+             PLUGIN_MODULE_VAR: self.getPlugin().getName()
              }
+
+        # Criteria in HostConfig.load to load or not QUEUE variables
+        if hc.getQueueSystem().hasName():
+            job_logs = self._getLogsPath(hc.getSubmitPrefix() + self.strId())
+            d['JOB_SCRIPT'] = job_logs + '.job'
+            d['JOB_LOGS'] = job_logs
+            d['JOB_NODEFILE'] = os.path.abspath(job_logs +'.nodefile')
+
         d.update(queueParams)
         return d
 
@@ -1946,12 +1949,12 @@ class Protocol(Step):
     def useQueueForSteps(self):
         """ This function will return True if the protocol has been set
         to be launched through a queue by steps """
-        return self.useQueue() and (self.getSubmitDict()["QUEUE_FOR_JOBS"] == "Y")
+        return self.useQueue() and (self.getSubmitDict()[QUEUE_FOR_JOBS] == "Y")
 
     def useQueueForProtocol(self):
         """ This function will return True if the protocol has been set
         to be launched through a queue """
-        return self.useQueue() and (self.getSubmitDict()["QUEUE_FOR_JOBS"] == "N")
+        return self.useQueue() and (self.getSubmitDict()[QUEUE_FOR_JOBS] != "Y")
 
     def getQueueParams(self):
         if self._queueParams.hasValue():
@@ -2441,6 +2444,11 @@ def runProtocolMain(projectPath, protDbPath, protId):
     setDefaultLoggingContext(protId, protocol.getProject().getShortName())
 
     hostConfig = protocol.getHostConfig()
+    gpuList = protocol.getGpuList()
+
+    #If queue is to be used
+    if protocol.useQueue():
+        gpuList = anonimizeGPUs(gpuList)
 
     # Create the steps executor
     executor = None
@@ -2451,18 +2459,18 @@ def runProtocolMain(projectPath, protDbPath, protId):
             executor = QueueStepExecutor(hostConfig,
                                          protocol.getSubmitDict(),
                                          nThreads - 1,
-                                         gpuList=protocol.getGpuList())
+                                         gpuList=gpuList)
         else:
             executor = ThreadStepExecutor(hostConfig, nThreads - 1,
-                                          gpuList=protocol.getGpuList())
+                                          gpuList=gpuList)
 
     if executor is None and protocol.useQueueForSteps():
         executor = QueueStepExecutor(hostConfig, protocol.getSubmitDict(), 1,
-                                     gpuList=protocol.getGpuList())
+                                     gpuList=gpuList)
 
     if executor is None:
         executor = StepExecutor(hostConfig,
-                                gpuList=protocol.getGpuList())
+                                gpuList=gpuList)
 
     logger.info("Running protocol using the %s executor." % executor)
     protocol.setStepsExecutor(executor)
@@ -2470,6 +2478,22 @@ def runProtocolMain(projectPath, protDbPath, protId):
     protocol.run()
 
 
+def anonimizeGPUs(gpuList):
+
+    renamedGPUs=dict()
+    anonimousGPUs = []
+
+    for gpu in gpuList:
+
+        if gpu not in renamedGPUs:
+            renamedGPUs[gpu] = len(renamedGPUs)
+
+        anonimousGPUs.append(renamedGPUs[gpu])
+
+    return anonimousGPUs
+
+
+
 def getProtocolFromDb(projectPath, protDbPath, protId, chdir=False):
     """ Retrieve the Protocol object from a given .sqlite file
     and the protocol id.
@@ -2535,6 +2559,7 @@ def isProtocolUpToDate(protocol):
 class ProtImportBase(Protocol):
     """ Base Import protocol"""
 
+
 class ProtStreamingBase(Protocol):
     """ Base protocol to implement streaming protocols.
     stepsGeneratorStep should be implemented (see its description) and output
@@ -2544,18 +2569,35 @@ class ProtStreamingBase(Protocol):
     """
 
     stepsExecutionMode = STEPS_PARALLEL
+
+    def _defineStreamingParams(self, form):
+        """ This function can be called during the _defineParams method
+        of some protocols that support stream processing.
+        It will add a Streaming section together with the following
+        params:
+            streamingSleepOnWait: Some streaming protocols are quite fast,
+                so, checking input/output updates creates an IO overhead.
+                This params allows them to sleep (without consuming resources)
+                to wait for new work to be done.
+        """
+        form.addSection("Streaming")
+        form.addParam("streamingSleepOnWait", IntParam, default=10,
+                      label="Sleep when waiting (secs)",
+                      help="If you specify a value greater than zero, "
+                           "it will be the number of seconds that the "
+                           "protocol will sleep when waiting for new "
+                           "input data in streaming mode. ")
+
     def _insertAllSteps(self):
-        # Insert the step that generates the steps
+        """ Insert the step that generates the steps """
         self._insertFunctionStep(self.resumableStepGeneratorStep, str(datetime.now()), needsGPU=False)
 
     def resumableStepGeneratorStep(self, ts):
-        """ This allow to resume protocols. ts is the time stamp so this stap is alway different form previous exceution"""
+        """ This allow to resume protocols. ts is the time stamp so this stap is always different form previous execution"""
         self.stepsGeneratorStep()
 
-
     def _stepsCheck(self):
-
-        # Just store steps created in checkNewInputStep
+        """ Just store steps created in checkNewInputStep"""
         if self._newSteps:
             self.updateSteps()
 
@@ -2569,11 +2611,28 @@ class ProtStreamingBase(Protocol):
         """
         pass
 
-    def _validateThreads(self, messages:list):
+    def _getStreamingSleepOnWait(self):
+        """ Retrieves the configured sleep duration for waiting during streaming.
+            Returns:
+            - int: The sleep duration in seconds during streaming wait.
+            """
+        return self.getAttributeValue('streamingSleepOnWait', 0)
+
+    def _streamingSleepOnWait(self):
+        """ This method should be used by protocols that want to sleep
+        when there is not more work to do.
+        """
+        sleepOnWait = self._getStreamingSleepOnWait()
+        if sleepOnWait > 0:
+            self.info("Waiting %s now before checking again for new input" % sleepOnWait)
+            time.sleep(sleepOnWait)
+
+    def _validateThreads(self, messages: list):
 
         if self.numberOfThreads.get() < 2:
             messages.append("At least 2 threads are needed for running this protocol. "
                             "1 for the 'stepsGenerator step' and one more for the actual processing" )
+
     def _validate(self):
         """ If you want to implement a validate method do it but call _validateThreads or validate threads value."""
         errors = []