scipion-pyworkflow 3.7.0__py3-none-any.whl

pyworkflow/project/scripts/stop.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# **************************************************************************
+# *
+# * Authors:     Yaiza Rancel (yrancel@cnb.csic.es)
+# *
+# * Unidad de Bioinformatica of Centro Nacional de Biotecnologia, CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+
+import sys
+import os
+
+import pyworkflow as pw
+import pyworkflow.utils as pwutils
+from pyworkflow.project import Manager, Project
+
+
+def usage(error):
+    print("""
+    ERROR: %s
+
+    Usage: scipion python -m pyworkflow.project.scripts.stop project_name
+        This script will stop all running protocols of the specified project.
+    """ % error)
+    sys.exit(1)
+
+
+n = len(sys.argv)
+
+if n > 2:
+    usage("This script accepts 1 mandatory parameter: the project name")
+
+projName = sys.argv[1]
+
+path = pw.join('gui', 'no-tkinter')
+sys.path.insert(1, path)
+
+manager = Manager()
+
+if not manager.hasProject(projName):
+    usage("There is no project with this name: %s"
+          % pwutils.red(projName))
+
+# the project may be a soft link which may be unavailable to the cluster so get the real path
+try:
+    projectPath = os.readlink(manager.getProjectPath(projName))
+except:
+    projectPath = manager.getProjectPath(projName)
+
+project = Project(pw.Config.getDomain(), projectPath)
+project.load()
+
+runs = project.getRuns()
+
+# Now assuming that there is no dependencies between runs
+# and the graph is linear
+for prot in runs:
+    if prot.isActive():
+        try:
+            project.stopProtocol(prot)
+        except:
+            print("Couldn't stop protocol %s" % prot)

pyworkflow/protocol/__init__.py

@@ -0,0 +1,38 @@
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
+# *
+# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+"""
+This modules contains classes required for the workflow
+execution and tracking like: Step and Protocol
+"""
+
+from .protocol import *
+from .executor import *
+from .constants import *
+from .params import *
+
+from .launch import *
+
+from .hosts import HostConfig

pyworkflow/protocol/bibtex.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# **************************************************************************
+# *
+# * Authors:     Yaiza Rancel (cyrancel@cnb.csic.es)
+# *
+# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+"""
+@article{delaRosaTrevin201693,
+title = "Scipion: A software framework toward integration, reproducibility and validation in 3D electron microscopy ",
+journal = "Journal of Structural Biology",
+volume = "195",
+number = "1",
+pages = "93 - 99",
+year = "2016",
+note = "",
+issn = "1047-8477",
+doi = "http://doi.org/10.1016/j.jsb.2016.04.010",
+url = "http://www.sciencedirect.com/science/article/pii/S104784771630079X",
+author = "J.M. de la Rosa-Trevín and A. Quintana and L. del Cano and A. Zaldívar and I. Foche and J. Gutiérrez and J. Gómez-Blanco and J. Burguet-Castell and J. Cuenca-Alba and V. Abrishami and J. Vargas and J. Otón and G. Sharov and J.L. Vilas and J. Navas and P. Conesa and M. Kazemi and R. Marabini and C.O.S. Sorzano and J.M. Carazo",
+keywords = "Electron microscopy",
+keywords = "Single particle analysis",
+keywords = "Image processing",
+keywords = "Software package",
+keywords = "Workflows",
+keywords = "Reproducibility "
+}
+"""
+

pyworkflow/protocol/constants.py

@@ -0,0 +1,86 @@
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
+# *
+# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+"""
+This modules contains classes required for the workflow
+execution and tracking like: Step and Protocol
+"""
+
+# ------------------ Constants values -----------------------------------------
+
+# Possible status of a protocol run, used mainly to monitor progress
+
+STATUS_SAVED = "saved"  # Parameters saved for later use
+STATUS_LAUNCHED = "launched"  # launched to queue system, only useful for protocols
+STATUS_NEW = "new"
+STATUS_RUNNING = "running"    # currently executing
+STATUS_FAILED = "failed"      # it run and failed
+STATUS_FINISHED = "finished"  # successfully finished
+STATUS_ABORTED = "aborted"
+STATUS_INTERACTIVE = "interactive"    # waiting for user interaction
+STATUS_WAITING = "waiting"            # following status is used for streaming only
+# Allow to schedule jobs not ready for running yet
+STATUS_SCHEDULED = "scheduled"
+
+ACTIVE_STATUS = [STATUS_LAUNCHED, STATUS_RUNNING, STATUS_INTERACTIVE,
+                 STATUS_SCHEDULED]
+
+# Execution modes
+MODE_RESUME = 0    # Try to starting at the first changed step, skipping unchanged ones
+MODE_RESTART = 1   # Restart the protocol from the beginning, deleting all previous results
+MODE_CONTINUE = 2  # Continue from specific step, not widely used //DEPRECATED.
+# JMRT: We now use 'Continue' label instead of 'Resume' which is more intuitive for users.
+MODE_CHOICES = ('Continue', 'Restart')  # , 'Continue')
+
+# Initial sleeping time (in seconds) in order to launch a scheduled protocol.
+# This value is multiply to the protocol level into the workflow.
+INITIAL_SLEEP_TIME = 30
+
+# Maximum time (in seconds) waiting for a scheduled protocol to check if it
+# can be launched.
+MAX_SLEEP_TIME = 120
+
+
+# Steps execution mode
+STEPS_SERIAL = 0      # Execute steps serially, some of the steps can be mpi programs
+STEPS_PARALLEL = 1    # Execute steps in parallel through threads
+
+# Level of expertise for the input parameters, mainly used in the protocol form
+LEVEL_NORMAL = 0
+LEVEL_ADVANCED = 1
+LEVEL_CHOICES = ('Normal', 'Advanced')
+
+# Param names for GPU processing
+USE_GPU = 'useGpu'
+GPU_LIST = 'gpuList'
+
+# Job management
+UNKNOWN_JOBID = -1
+
+# File sizes
+SIZE_1KB = 1024
+SIZE_1MB = SIZE_1KB * SIZE_1KB
+SIZE_1GB = SIZE_1MB * SIZE_1KB
+SIZE_1TB = SIZE_1GB * SIZE_1KB

pyworkflow/protocol/executor.py

@@ -0,0 +1,334 @@
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
+# *
+# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia, CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+"""
+This module have the classes for execution of protocol steps.
+The basic one will run steps, one by one, after completion.
+There is one based on threads to execute steps in parallel
+using different threads and the last one with MPI processes.
+"""
+
+import logging
+logger = logging.getLogger(__name__)
+import time
+import datetime
+import threading
+import os
+
+import pyworkflow.utils.process as process
+from pyworkflow.utils.path import getParentFolder, removeExt
+from . import constants as cts
+
+from .launch import _submit, UNKNOWN_JOBID, _checkJobStatus
+
+
+class StepExecutor:
+    """ Run a list of Protocol steps. """
+    def __init__(self, hostConfig, **kwargs):
+        self.hostConfig = hostConfig
+        self.gpuList = kwargs.get(cts.GPU_LIST, None)
+        self.protocol = None
+
+    def getGpuList(self):
+        """ Return the GPU list assigned to current thread. """
+        return self.gpuList
+
+    def setProtocol(self, protocol):
+        """ Set protocol to append active jobs to its jobIds. """
+        self.protocol = protocol
+
+    def runJob(self, log, programName, params,           
+               numberOfMpi=1, numberOfThreads=1,
+               env=None, cwd=None, executable=None):
+        """ This function is a wrapper around runJob, 
+        providing the host configuration. 
+        """
+        process.runJob(log, programName, params,
+                       numberOfMpi, numberOfThreads, 
+                       self.hostConfig,
+                       env=env, cwd=cwd, gpuList=self.getGpuList(), executable=executable)
+        
+    def _getRunnable(self, steps, n=1):
+        """ Return the n steps that are 'new' and all its
+        dependencies have been finished, or None if none ready.
+        """
+        rs = []  # return a list of runnable steps
+
+        for s in steps:
+            if (s.getStatus() == cts.STATUS_NEW and
+                    all(steps[i-1].isFinished() for i in s._prerequisites)):
+                rs.append(s)
+                if len(rs) == n:
+                    break
+        return rs
+    
+    def _arePending(self, steps):
+        """ Return True if there are pending steps (either running or waiting)
+        that can be done and thus enable other steps to be executed.
+        """
+        return any(s.isRunning() or s.isWaiting() for s in steps)
+    
+    def runSteps(self, steps, 
+                 stepStartedCallback, 
+                 stepFinishedCallback,
+                 stepsCheckCallback,
+                 stepsCheckSecs=3):
+        # Even if this will run the steps in a single thread
+        # let's follow a similar approach than the parallel one
+        # In this way we can take into account the steps graph
+        # dependency and also the case when using streaming
+
+        delta = datetime.timedelta(seconds=stepsCheckSecs)
+        lastCheck = datetime.datetime.now()
+
+        while True:
+            # Get a step to run, if there is any
+            runnableSteps = self._getRunnable(steps)
+
+            if runnableSteps:
+                step = runnableSteps[0]
+                # We found a step to work in, so let's start a new
+                # thread to do the job and book it.
+                step.setRunning()
+                stepStartedCallback(step)
+                step.run()
+                doContinue = stepFinishedCallback(step)
+            
+                if not doContinue:
+                    break
+
+            elif self._arePending(steps):
+                # We have not found any runnable step, but still there
+                # there are some running or waiting for dependencies
+                # So, let's wait a bit to check if something changes
+                time.sleep(0.5)
+            else:
+                # No steps to run, neither running or waiting
+                # So, we are done, either failed or finished :)
+                break
+
+            now = datetime.datetime.now()
+            if now - lastCheck > delta:
+                stepsCheckCallback()
+                lastCheck = now
+
+        stepsCheckCallback()  # one last check to finalize stuff
+
+
+class StepThread(threading.Thread):
+    """ Thread to run Steps in parallel. """
+    def __init__(self, thId, step, lock):
+        threading.Thread.__init__(self)
+        self.thId = thId
+        self.step = step
+        self.lock = lock
+
+    def run(self):
+        error = None
+        try:
+            self.step._run()  # not self.step.run() , to avoid race conditions
+        except Exception as e:
+            error = str(e)
+            logger.error("Couldn't run the code in a thread." , exc_info=e)
+        finally:
+            with self.lock:
+                if error is None:
+                    self.step.setFinished()
+                else:
+                    self.step.setFailed(error)
+
+
+class ThreadStepExecutor(StepExecutor):
+    """ Run steps in parallel using threads. """
+    def __init__(self, hostConfig, nThreads, **kwargs):
+        StepExecutor.__init__(self, hostConfig, **kwargs)
+        self.numberOfProcs = nThreads
+        # If the gpuList was specified, we need to distribute GPUs among
+        # all the threads
+        self.gpuDict = {}
+
+        if self.gpuList:
+            nodes = range(nThreads)
+            nGpu = len(self.gpuList)
+
+            if nGpu > nThreads:
+                chunk = int(nGpu / nThreads)
+                for i, node in enumerate(nodes):
+                    self.gpuDict[node] = list(self.gpuList[i*chunk:(i+1)*chunk])
+            else:
+                # Expand gpuList repeating until reach nThreads items
+                if nThreads > nGpu:
+                    newList = self.gpuList * (int(nThreads/nGpu)+1)
+                    self.gpuList = newList[:nThreads]
+
+                for node, gpu in zip(nodes, self.gpuList):
+                    self.gpuDict[node] = [gpu]
+
+    def getGpuList(self):
+        """ Return the GPU list assigned to current thread
+        or empty list if not using GPUs. """
+        return self.gpuDict.get(threading.current_thread().thId, [])
+        
+    def runSteps(self, steps, 
+                 stepStartedCallback, 
+                 stepFinishedCallback,
+                 stepsCheckCallback,
+                 stepsCheckSecs=5):
+        """
+        Creates threads and synchronize the steps execution.
+
+        :param steps: list of steps to run
+        :param stepStartedCallback: callback to be called before starting any step
+        :param stepFinishedCallback: callback to be run after all steps are done
+        :param stepsCheckCallback: callback to check if there are new steps to add (streaming)
+        :param stepsCheckSecs: seconds between stepsCheckCallback calls
+
+        """
+
+        delta = datetime.timedelta(seconds=stepsCheckSecs)
+        lastCheck = datetime.datetime.now()
+
+        sharedLock = threading.Lock()
+
+        runningSteps = {}  # currently running step in each node ({node: step})
+        freeNodes = list(range(self.numberOfProcs))  # available nodes to send jobs
+
+        while True:
+            # See which of the runningSteps are not really running anymore.
+            # Update them and freeNodes, and call final callback for step.
+            with sharedLock:
+                nodesFinished = [node for node, step in runningSteps.items()
+                                 if not step.isRunning()]
+            doContinue = True
+            for node in nodesFinished:
+                step = runningSteps.pop(node)  # remove entry from runningSteps
+                freeNodes.append(node)  # the node is available now
+                # Notify steps termination and check if we should continue
+                doContinue = stepFinishedCallback(step)
+                if not doContinue:
+                    break
+
+            if not doContinue:
+                break
+
+            anyLaunched = False
+            # If there are available nodes, send next runnable step.
+            with sharedLock:
+                if freeNodes:
+                    runnableSteps = self._getRunnable(steps, len(freeNodes))
+
+                    for step in runnableSteps:
+                        # We found a step to work in, so let's start a new
+                        # thread to do the job and book it.
+                        anyLaunched = True
+                        step.setRunning()
+                        stepStartedCallback(step)
+                        node = freeNodes.pop()  # take an available node
+                        runningSteps[node] = step
+                        t = StepThread(node, step, sharedLock)
+                        # won't keep process up if main thread ends
+                        t.daemon = True
+                        t.start()
+                anyPending = self._arePending(steps)
+
+            if not anyLaunched:
+                if anyPending:  # nothing running
+                    time.sleep(0.5)
+                else:
+                    break  # yeah, we are done, either failed or finished :)
+
+            now = datetime.datetime.now()
+            if now - lastCheck > delta:
+                stepsCheckCallback()
+                lastCheck = now
+
+        stepsCheckCallback()
+
+        # Wait for all threads now.
+        for t in threading.enumerate():
+            if t is not threading.current_thread():
+                t.join()
+
+
+class QueueStepExecutor(ThreadStepExecutor):
+    def __init__(self, hostConfig, submitDict, nThreads, **kwargs):
+        ThreadStepExecutor.__init__(self, hostConfig, nThreads, **kwargs)
+        self.submitDict = submitDict
+        # Command counter per thread
+        self.threadCommands = {}
+        for threadId in range(nThreads):
+            self.threadCommands[threadId] = 0
+
+        if nThreads > 1:
+            self.runJobs = ThreadStepExecutor.runSteps
+        else:
+            self.runJobs = StepExecutor.runSteps
+
+        self.renameGpuIds()
+
+    def renameGpuIds(self):
+        """ Reorganize the gpus ids starting from 0 since the queue engine is the one assigning them.
+        https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars """
+        for threadId, gpuList in self.gpuDict.items():
+            for i in range(len(gpuList)):
+                self.gpuDict[threadId][i] = i
+
+        logger.debug("Updated gpus ids rebase starting from 0: %s per thread" %self.gpuDict)
+
+    def runJob(self, log, programName, params, numberOfMpi=1, numberOfThreads=1, env=None, cwd=None, executable=None):
+        threadId = threading.current_thread().thId
+        submitDict = dict(self.hostConfig.getQueuesDefault())
+        submitDict.update(self.submitDict)
+        submitDict['JOB_COMMAND'] = process.buildRunCommand(programName, params, numberOfMpi,
+                                                            self.hostConfig, env,
+                                                            gpuList=self.getGpuList())
+        self.threadCommands[threadId] += 1
+        subthreadId = '-%s-%s' % (threadId, self.threadCommands[threadId])
+        submitDict['JOB_NAME'] = submitDict['JOB_NAME'] + subthreadId
+        submitDict['JOB_SCRIPT'] = os.path.abspath(removeExt(submitDict['JOB_SCRIPT']) + subthreadId + ".job")
+        submitDict['JOB_LOGS'] = os.path.join(getParentFolder(submitDict['JOB_SCRIPT']), submitDict['JOB_NAME'])
+
+        jobid = _submit(self.hostConfig, submitDict, cwd, env)
+        self.protocol.appendJobId(jobid)  # append active jobs
+        self.protocol._store(self.protocol._jobId)
+
+        if (jobid is None) or (jobid == UNKNOWN_JOBID):
+            logger.info("jobId is none therefore we set it to fail")
+            raise Exception("Failed to submit to queue.")
+
+        status = cts.STATUS_RUNNING
+        wait = 3
+
+        # Check status while job running
+        # REVIEW this to minimize the overhead in time put by this delay check
+        while _checkJobStatus(self.hostConfig, jobid) == cts.STATUS_RUNNING:
+            time.sleep(wait)
+            if wait < 300:
+                wait += 3
+
+        self.protocol.removeJobId(jobid)  # After completion, remove inactive jobs.
+        self.protocol._store(self.protocol._jobId)
+
+        return status