scipion-pyworkflow 3.10.6__py3-none-any.whl → 3.11.0__py3-none-any.whl

pyworkflow/protocol/hosts.py

@@ -1,314 +0,0 @@
-# **************************************************************************
-# *
-# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
-# *
-# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
-# *
-# * This program is free software; you can redistribute it and/or modify
-# * it under the terms of the GNU General Public License as published by
-# * the Free Software Foundation; either version 3 of the License, or
-# * (at your option) any later version.
-# *
-# * This program is distributed in the hope that it will be useful,
-# * but WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# * GNU General Public License for more details.
-# *
-# * You should have received a copy of the GNU General Public License
-# * along with this program; if not, write to the Free Software
-# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# * 02111-1307  USA
-# *
-# *  All comments concerning this program package may be sent to the
-# *  e-mail address 'scipion@cnb.csic.es'
-# *
-# **************************************************************************
-"""
-This modules contains classes to store information about
-execution hosts.
-"""
-
-
-import os
-import sys
-import json
-from configparser import RawConfigParser
-from collections import OrderedDict
-
-import pyworkflow as pw
-from pyworkflow import PARALLEL_COMMAND_VAR
-from pyworkflow.object import Object, String, Integer
-
-
-class HostConfig(Object):
-    """ Main store the configuration for execution hosts. """
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.label = String(kwargs.get('label', None))
-        self.hostName = String(kwargs.get('hostName', None))
-        self.userName = String()
-        self.password = String()
-        self.hostPath = String()
-        self.mpiCommand = String()
-        self.scipionHome = String()
-        self.scipionConfig = String()
-        self.address = String()
-        self.queueSystem = QueueSystemConfig()
-
-    def getLabel(self):
-        return self.label.get()
-
-    def getHostName(self):
-        return self.hostName.get()
-
-    def getUserName(self):
-        return self.userName.get()
-
-    def getPassword(self):
-        return self.password.get()
-
-    def getHostPath(self):
-        return self.hostPath.get()
-
-    def getSubmitCommand(self):
-        return self.queueSystem.submitCommand.get()
-
-    def getSubmitPrefix(self):
-        return self.queueSystem.submitPrefix.get()
-
-    def getCheckCommand(self):
-        return self.queueSystem.checkCommand.get()
-
-    def getCancelCommand(self):
-        return self.queueSystem.cancelCommand.get()
-
-    def isQueueMandatory(self):
-        return self.queueSystem.mandatory.get()
-
-    def getSubmitTemplate(self):
-        return self.queueSystem.getSubmitTemplate()
-
-    def getQueuesDefault(self):
-        return self.queueSystem.queuesDefault
-
-    def getMpiCommand(self):
-        return self.mpiCommand.get()
-
-    def getQueueSystem(self):
-        return self.queueSystem
-
-    def getJobDoneRegex(self):
-        return self.queueSystem.jobDoneRegex.get()
-
-    def setLabel(self, label):
-        self.label.set(label)
-
-    def setHostName(self, hostName):
-        self.hostName.set(hostName)
-
-    def setUserName(self, userName):
-        self.userName.set(userName)
-
-    def setPassword(self, password):
-        self.password.set(password)
-
-    def setHostPath(self, hostPath):
-        self.hostPath.set(hostPath)
-
-    def setMpiCommand(self, mpiCommand):
-        self.mpiCommand.set(mpiCommand)
-
-    def setQueueSystem(self, queueSystem):
-        self.queueSystem = queueSystem
-
-    def getScipionHome(self):
-        """ Return the path where Scipion is installed in
-        the host.
-        """
-        return self.scipionHome.get()
-
-    def setScipionHome(self, newScipionHome):
-        self.scipionHome.set(newScipionHome)
-
-    def getScipionConfig(self):
-        """ From which file to read the configuration file in
-        this hosts.
-        """
-        return self.scipionConfig.get()
-
-    def setScipionConfig(self, newConfig):
-        self.scipionConfig.set(newConfig)
-
-    def getAddress(self):
-        return self.address.get()
-
-    def setAddress(self, newAddress):
-        return self.address.set(newAddress)
-
-    @classmethod
-    def writeBasic(cls, configFn):
-        """ Write a very basic Host configuration for testing purposes. """
-        with open(configFn, 'w') as f:
-            f.write('[localhost]\nPARALLEL_COMMAND = '
-                    'mpirun -np %_(JOB_NODES)d --map-by node %_(COMMAND)s\n')
-
-    @classmethod
-    def load(cls, hostsConf):
-        """ Load several hosts from a configuration file.
-        Return an dictionary with hostName -> hostConfig pairs.
-        """
-        # Read from users' config file. Raw to avoid interpolation of %: we expect %_
-        cp = RawConfigParser(comment_prefixes=";")
-        cp.optionxform = str  # keep case (stackoverflow.com/questions/1611799)
-        hosts = OrderedDict()
-
-        try:
-            assert cp.read(hostsConf) != [], 'Missing file %s' % hostsConf
-
-            for hostName in cp.sections():
-                host = HostConfig(label=hostName, hostName=hostName)
-                host.setHostPath(pw.Config.SCIPION_USER_DATA)
-
-                # Helper functions (to write less)
-                def get(var, default=None):
-                    if cp.has_option(hostName, var):
-
-                        value = cp.get(hostName, var)
-                        # Rescue python2.7 behaviour: ## at the beginning of a line, means a single #.
-                        # https://github.com/scipion-em/scipion-pyworkflow/issues/70
-                        value = value.replace("\n##", "\n#")
-
-                        # Keep compatibility: %_ --> %%
-                        value = value.replace('%_(', '%(')
-
-                        return value
-                    else:
-                        return default
-
-                def getDict(var):
-                    od = OrderedDict()
-
-                    if cp.has_option(hostName, var):
-                        for key, value in json.loads(get(var)).items():
-                            od[key] = value
-
-                    return od
-
-                host.setScipionHome(get(pw.SCIPION_HOME_VAR, pw.Config.SCIPION_HOME))
-                host.setScipionConfig(pw.Config.SCIPION_CONFIG)
-                # Read the address of the remote hosts,
-                # using 'localhost' as default for backward compatibility
-                host.setAddress(get('ADDRESS', 'localhost'))
-                host.mpiCommand.set(get(PARALLEL_COMMAND_VAR))
-                host.queueSystem = QueueSystemConfig()
-                hostQueue = host.queueSystem  # shortcut
-                hostQueue.name.set(get('NAME'))
-
-                # If the NAME is not provided or empty
-                # do no try to parse the rest of Queue parameters
-                if hostQueue.hasName():
-                    hostQueue.setMandatory(get('MANDATORY', 0))
-                    hostQueue.submitPrefix.set(get('SUBMIT_PREFIX', ''))
-                    hostQueue.submitCommand.set(get('SUBMIT_COMMAND'))
-                    hostQueue.submitTemplate.set(get('SUBMIT_TEMPLATE'))
-                    hostQueue.cancelCommand.set(get('CANCEL_COMMAND'))
-                    hostQueue.checkCommand.set(get('CHECK_COMMAND'))
-                    hostQueue.jobDoneRegex.set(get('JOB_DONE_REGEX'))
-                    hostQueue.queues = getDict('QUEUES')
-                    hostQueue.queuesDefault = getDict('QUEUES_DEFAULT')
-
-                hosts[hostName] = host
-
-            return hosts
-        except Exception as e:
-            sys.exit('Failed to read settings. The reported error was:\n  %s\n'
-                     'Review %s and run again.'
-                     % (e, os.path.abspath(hostsConf)))
-
-
-class QueueSystemConfig(Object):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.name = String()
-        # Number of cores from which the queue is mandatory
-        # 0 means no mandatory at all
-        # 1 will force to launch all jobs through the queue
-        self.mandatory = Integer()
-        self.queues = None  # List for queue configurations
-        self.submitCommand = String()
-        # Allow to change the prefix of submission scripts
-        # we used by default the ID.job, but in some clusters
-        # the job script should start by a letter
-        self.submitPrefix = String()
-        self.checkCommand = String()
-        self.cancelCommand = String()
-        self.submitTemplate = String()
-        self.jobDoneRegex = String()
-
-    def hasName(self):
-        return self.name.hasValue()
-
-    def hasValue(self):
-        return self.hasName() and len(self.queues)
-
-    def getName(self):
-        return self.name.get()
-
-    def getMandatory(self):
-        return self.mandatory.get()
-
-    def getSubmitTemplate(self):
-        return self.submitTemplate.get()
-
-    def getSubmitCommand(self):
-        return self.submitCommand.get()
-
-    def getCheckCommand(self):
-        return self.checkCommand.get()
-
-    def getCancelCommand(self):
-        return self.cancelCommand.get()
-
-    def getQueues(self):
-        return self.queues
-
-    def setName(self, name):
-        self.name.set(name)
-
-    def setMandatory(self, mandatory):
-        # This condition is to be backward compatible
-        # when mandatory was a boolean
-        # now it should use the number of CPU
-        # that should force to use the queue
-        if mandatory in ['False', 'false']:
-            mandatory = 0
-        elif mandatory in ['True', 'true']:
-            mandatory = 1
-
-        self.mandatory.set(mandatory)
-
-    def setSubmitTemplate(self, submitTemplate):
-        self.submitTemplate.set(submitTemplate)
-
-    def setSubmitCommand(self, submitCommand):
-        self.submitCommand.set(submitCommand)
-
-    def setCheckCommand(self, checkCommand):
-        self.checkCommand.set(checkCommand)
-
-    def setCancelCommand(self, cancelCommand):
-        self.cancelCommand.set(cancelCommand)
-
-    def setJobDoneRegex(self, jobDoneRegex):
-        self.jobDoneRegex.set(jobDoneRegex)
-
-    def setQueues(self, queues):
-        self.queues = queues
-
-    def getQueueConfig(self, objId):
-        if objId is not None and self.queues is not None:
-            for queueConfig in self.queues:
-                if objId == queueConfig.getObjId():
-                    return queueConfig
-        return None

pyworkflow/protocol/launch.py

@@ -1,270 +0,0 @@
-# **************************************************************************
-# *
-# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
-# *
-# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
-# *
-# * This program is free software; you can redistribute it and/or modify
-# * it under the terms of the GNU General Public License as published by
-# * the Free Software Foundation; either version 3 of the License, or
-# * (at your option) any later version.
-# *
-# * This program is distributed in the hope that it will be useful,
-# * but WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# * GNU General Public License for more details.
-# *
-# * You should have received a copy of the GNU General Public License
-# * along with this program; if not, write to the Free Software
-# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# * 02111-1307  USA
-# *
-# *  All comments concerning this program package may be sent to the
-# *  e-mail address 'scipion@cnb.csic.es'
-# *
-# **************************************************************************
-"""
-This module is responsible for launching local protocol executions:
-
-1- Check if the protocol will be submitted to a queue (using Queue template from config)
-2- Build the command that will be launched.
-
-"""
-
-import os
-import re
-import logging
-logger = logging.getLogger(__file__)
-from subprocess import Popen, PIPE
-import pyworkflow as pw
-from pyworkflow.exceptions import PyworkflowException
-from pyworkflow.utils import (redStr, greenStr, makeFilePath, join, process,
-                              getHostFullName)
-from pyworkflow.protocol.constants import UNKNOWN_JOBID, STATUS_FAILED, STATUS_FINISHED, STATUS_RUNNING
-
-
-# ******************************************************************
-# *         Public functions provided by the module
-# ******************************************************************
-
-def launch(protocol, wait=False, stdin=None, stdout=None, stderr=None):
-    """ This function should be used to launch a protocol. """
-    _launchLocal(protocol, wait, stdin, stdout, stderr)
-
-
-def stop(protocol):
-    """
-    Stop function for three scenarios:
-    - If the queue is not used, kill the main protocol process and its child processes.
-    - If the queue is used and the entire protocol is sent to the queue, cancel the job running the protocol using
-    scancel.
-    - If the queue is used and individual steps are sent to the queue, cancel all active jobs and kill the main protocol
-    process and its child processes.
-    """
-    if protocol.useQueue() and not protocol.isScheduled():
-        jobIds = protocol.getJobIds()
-        for jobId in jobIds: # Iter even though it contains only one jobId
-            host = protocol.getHostConfig()
-            cancelCmd = host.getCancelCommand() % {'JOB_ID': jobId}
-            logger.info(cancelCmd)
-            _run(cancelCmd, wait=True)
-
-        if protocol.useQueueForSteps():
-            process.killWithChilds(protocol.getPid())
-    else:
-        process.killWithChilds(protocol.getPid())
-
-
-def schedule(protocol, initialSleepTime=0, wait=False):
-    """ Use this function to schedule protocols that are not ready to
-    run yet. Right now it only make sense to schedule jobs locally.
-    """
-    cmd = '%s %s' % (pw.PYTHON, pw.getScheduleScript())
-    cmd += ' "%s" "%s" %s "%s" --initial_sleep %s' % (protocol.getProject().path,
-                              protocol.getDbPath(),
-                              protocol.strId(),
-                              protocol.getScheduleLog(),
-                              initialSleepTime)
-    pid = _run(cmd, wait)
-    protocol.setPid(pid)  # Set correctly the pid
-
-
-# ******************************************************************
-# *         Internal utility functions
-# ******************************************************************
-def _runsLocally(protocol):
-    """ Return True if this protocol is running in this machine,
-    where the PID makes sense.
-    """
-    return protocol.getHostFullName() == getHostFullName()
-
-
-# ******************************************************************
-# *                 Function related to LAUNCH
-# ******************************************************************
-def _getAppsProgram(prog):
-    """ Get a command to launch a program under the apps folder.
-    """
-    return "%s %s" % (pw.PYTHON, pw.join(pw.APPS, prog))
-
-
-def _launchLocal(protocol, wait, stdin=None, stdout=None, stderr=None):
-    """
-
-    :param protocol: Protocol to launch
-    :param wait: Pass true if you want to wait for the process to finish
-    :param stdin: stdin object to direct stdin to
-    :param stdout: stdout object to send process stdout
-    :param stderr: stderr object to send process stderr
-    """
-
-    command = '{python} {prot_run} "{project_path}" "{db_path}" {prot_id} "{stdout_log}" "{stderr_log}"'.format(
-        python=pw.PYTHON,
-        prot_run=pw.join(pw.APPS, 'pw_protocol_run.py'),
-        project_path=protocol.getProject().path,
-        db_path=protocol.getDbPath(),
-        prot_id=protocol.strId(),
-        # We make them absolute in case working dir is not passed to the node when running through a queue.
-        # The reason is that since 3.0.27, the first thing that is affected by the current working dir is the
-        # creation of the logs. Before event than loading the project, which was and is setting the working dir to
-        # the project path. IMPORTANT: This assumes the paths before the queue and after the queue (nodes) are the same
-        # Which I think is safe since we are passing here "project_path" that is absolute.
-        stdout_log=os.path.abspath(protocol.getStdoutLog()),
-        stderr_log=os.path.abspath(protocol.getStderrLog())
-    )
-
-    hostConfig = protocol.getHostConfig()
-
-    # Clean Pid and JobIds
-    protocol.cleanExecutionAttributes()
-    protocol._store(protocol._jobId)
-
-    # Handle three use cases: one will use the job ID, and the other two will use the process ID.
-    if protocol.useQueueForProtocol():  # Retrieve the job ID and set it; this will be used to control the protocol.
-        submitDict = dict(hostConfig.getQueuesDefault())
-        submitDict.update(protocol.getSubmitDict())
-        submitDict['JOB_COMMAND'] = command
-        jobId = _submit(hostConfig, submitDict)
-        if jobId is None or jobId == UNKNOWN_JOBID:
-            protocol.setStatus(STATUS_FAILED)
-        else:
-            protocol.setJobId(jobId)
-            protocol.setPid(0)  # we go through the queue, so we rely on the jobId
-    else:  # If not, retrieve and set the process ID (both for normal execution or when using the queue for steps)
-        pId = _run(command, wait, stdin, stdout, stderr)
-        protocol.setPid(pId)
-
-
-def analyzeFormattingTypeError(string, dictionary):
-    """ receives a string with %(VARS) to be replaced with a dictionary
-     it splits te string by \n and test the formatting per line. Raises an exception if any line fails
-     with all problems found"""
-
-    # Do the replacement line by line
-    lines = string.split("\n")
-
-    problematicLines = []
-    for line in lines:
-        try:
-            line % dictionary
-        except KeyError as e:
-            problematicLines.append(line + " --> variable not present in this context.")
-        except Exception as e:
-            problematicLines.append(line + " --> " + str(e))
-
-    if problematicLines:
-        return PyworkflowException('Following lines in %s seems to be problematic.\n'
-                                   'Values known in this context are: \n%s'
-                                   'Please review its format or content.\n%s' % (dictionary, pw.Config.SCIPION_HOSTS, "\n".join(problematicLines)),
-                                   url=pw.DOCSITEURLS.HOST_CONFIG)
-
-
-def _submit(hostConfig, submitDict, cwd=None, env=None):
-    """ Submit a protocol to a queue system. Return its job id.
-    """
-    # Create first the submission script to be launched
-    # formatting using the template
-    template = hostConfig.getSubmitTemplate()
-
-    try:
-        template = template % submitDict
-    except Exception as e:
-        # Capture parsing errors
-        exception = analyzeFormattingTypeError(template, submitDict)
-
-        if exception:
-            raise exception
-        else:
-            # If there is no exception, then raise actual one
-            raise e
-
-    # FIXME: CREATE THE PATH FIRST
-    scripPath = submitDict['JOB_SCRIPT']
-    f = open(scripPath, 'w')
-    # Ensure the path exists
-    makeFilePath(scripPath)
-    # Add some line ends because in some clusters it fails
-    # to submit jobs if the submit script does not have end of line
-    f.write(template + '\n\n')
-    f.close()
-    # This should format the command using a template like: 
-    # "qsub %(JOB_SCRIPT)s"
-    command = hostConfig.getSubmitCommand() % submitDict
-    gcmd = greenStr(command)
-    logger.info("** Submitting to queue: '%s'" % gcmd)
-
-    p = Popen(command, shell=True, stdout=PIPE, cwd=cwd, env=env)
-    out = p.communicate()[0]
-    # Try to parse the result of qsub, searching for a number (jobId)
-    # Review this, seems to exclusive to torque batch system
-    s = re.search(r'(\d+)', str(out))
-    if p.returncode == 0 and s:
-        job = int(s.group(0))
-        logger.info("Launched job with id %s" % job)
-        return job
-    else:
-        logger.info("Couldn't submit to queue for reason: %s " % redStr(out.decode()))
-        return UNKNOWN_JOBID
-
-def _checkJobStatus(hostConfig, jobid):
-    """
-    General method to verify the job status in the queue based on the jobId and host.conf CHECK_COMMAND
-    returns: STATUS_FINISHED (finished) or STATUS_RUNNING (running)
-    """
-    command = hostConfig.getCheckCommand() % {"JOB_ID": jobid}
-    logger.debug("checking job status for %s: %s" % (jobid, command))
-
-    p = Popen(command, shell=True, stdout=PIPE, preexec_fn=os.setsid)
-
-    out = p.communicate()[0].decode(errors='backslashreplace')
-
-    jobDoneRegex = hostConfig.getJobDoneRegex()
-    logger.debug("Queue engine replied %s, variable JOB_DONE_REGEX has %s" % (out, jobDoneRegex))
-    # If nothing is returned we assume job is no longer in queue and thus finished
-    if out == "":
-        logger.warning("Empty response from queue system to job (%s)" % jobid)
-        return STATUS_FINISHED
-
-    # If some string is returned we use the JOB_DONE_REGEX variable (if present) to infer the status
-    elif jobDoneRegex is not None:
-        s = re.search(jobDoneRegex, out)
-        if s:
-            logger.debug("Job (%s) finished" % jobid)
-            return STATUS_FINISHED
-        else:
-            logger.debug("Job (%s) still running" % jobid)
-            return STATUS_RUNNING
-    # If JOB_DONE_REGEX is not defined and queue has returned something we assume that job is still running
-    else:
-        return STATUS_RUNNING
-
-def _run(command, wait, stdin=None, stdout=None, stderr=None):
-    """ Execute a command in a subprocess and return the pid. """
-    gcmd = greenStr(command)
-    logger.info("** Running command: '%s'" % gcmd)
-    p = Popen(command, shell=True, stdout=stdout, stderr=stderr)
-    pid = p.pid
-    if wait:
-        p.wait()
-
-    return pid

pyworkflow/protocol/package.py

@@ -1,42 +0,0 @@
-# **************************************************************************
-# *
-# * Authors:     P. Conesa Mingo (pconesa@cnb.csic.es)
-# *
-# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
-# *
-# * This program is free software; you can redistribute it and/or modify
-# * it under the terms of the GNU General Public License as published by
-# * the Free Software Foundation; either version 3 of the License, or
-# * (at your option) any later version.
-# *
-# * This program is distributed in the hope that it will be useful,
-# * but WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# * GNU General Public License for more details.
-# *
-# * You should have received a copy of the GNU General Public License
-# * along with this program; if not, write to the Free Software
-# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# * 02111-1307  USA
-# *
-# *  All comments concerning this program package may be sent to the
-# *  e-mail address 'scipion@cnb.csic.es'
-# *
-# **************************************************************************
-
-
-"""
-This modules contains classes related to EM package management
-"""
-
-
-class Package:
-    def __init__(self, name, installed):
-        self._name = name
-        self._installed = installed
-
-    def isInstalled(self):
-        return self._installed
-
-    def getName(self):
-        return self._name