scipion-pyworkflow 3.10.6__py3-none-any.whl → 3.11.0__py3-none-any.whl

pyworkflow/apps/pw_run_tests.py

@@ -1,267 +0,0 @@
-#!/usr/bin/env python
-# **************************************************************************
-# *
-# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
-# *
-# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
-# *
-# * This program is free software; you can redistribute it and/or modify
-# * it under the terms of the GNU General Public License as published by
-# * the Free Software Foundation; either version 3 of the License, or
-# * (at your option) any later version.
-# *
-# * This program is distributed in the hope that it will be useful,
-# * but WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# * GNU General Public License for more details.
-# *
-# * You should have received a copy of the GNU General Public License
-# * along with this program; if not, write to the Free Software
-# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# * 02111-1307  USA
-# *
-# *  All comments concerning this program package may be sent to the
-# *  e-mail address 'scipion@cnb.csic.es'
-# *
-# **************************************************************************
-
-"""
-Run or show the selected tests. Tests can be selected by giving
-the "case", or by giving the paths and file pattern to use for
-searching them.
-"""
-from pyworkflow.utils import LoggingConfigurator
-import argparse
-from collections import OrderedDict
-
-import pyworkflow.tests as pwtests
-from pyworkflow import getTestsScript, SCIPION_TESTS_CMD, Config
-
-from pyworkflow.tests import *
-
-MODULE = 0
-CLASS = 1
-TEST = 2
-
-
-class Tester:
-    def main(self, args=None):
-
-        # Trigger plugin's variable definition
-        Config.getDomain().getPlugins()
-
-        parser = argparse.ArgumentParser(prog=self.getTestsCommand(), description=__doc__)
-        g = parser.add_mutually_exclusive_group()
-        g.add_argument('--run', action='store_true', help='run the selected tests')
-        g.add_argument('--show', action='store_true', help='show available tests',
-                       default=True)
-
-        add = parser.add_argument  # shortcut
-
-        add('--pattern', default='test*.py',
-            help='pattern for the files that will be used in the tests')
-        add('--grep', default=None, nargs='+',
-            help='only show/run tests containing the provided words')
-        add('--skip', default=None, nargs='+',
-            help='skip tests that contains these words')
-        add('--log', default=None, nargs='?',
-            help="Generate logs files with the output of each test.")
-        add('--mode', default='classes', choices=['modules', 'classes', 'onlyclasses', 'all'],
-            help='how much detail to give in show mode')
-        add('tests', metavar='TEST', nargs='*',
-            help='test case from string identifier (module, class or callable)')
-        args = parser.parse_args(args)
-
-        if not args.run and not args.show and not args.tests:
-            sys.exit(parser.format_help())
-
-        # Logging stuff first
-        self.log = args.log
-
-        if self.log:
-            LoggingConfigurator.setupLogging(logFile=self.log)
-        else:
-            logging.basicConfig(level=Config.SCIPION_LOG_LEVEL, format=Config.SCIPION_LOG_FORMAT)
-
-        self.logger = logging.getLogger(__name__)
-
-        # This goes intentionally to the output. Is not a logging line._S
-        print("Scanning tests...")
-
-
-        testsDict = OrderedDict()
-        testLoader = unittest.defaultTestLoader
-
-        if args.tests:
-            print(pwutils.cyanStr("Loading test/s %s" % args.tests))
-
-            # In this case the tests are passed as argument.
-            # The full name of the test will be used to load it.
-            testsDict['tests'] = unittest.TestSuite()
-            for t in args.tests:
-                try:
-                    testsDict['tests'].addTests(testLoader.loadTestsFromName(t))
-                except Exception as e:
-                    self.logger.error('Cannot load test %s -- skipping' % t, exc_info=True)
-        else:
-            # In this other case, we will load the test available
-            # from pyworkflow and the other plugins
-            # self.paths = [('pyworkflow', os.path.dirname(os.path.dirname(pw.__file__)))]
-            self.paths = []
-            for name, plugin in pw.Config.getDomain().getPlugins().items():
-                self.paths.append((name, os.path.dirname(plugin.__path__[0])))
-            for k, p in self.paths:
-                testPath = os.path.join(p, k, 'tests')
-                if os.path.exists(testPath):
-                    self.logger.debug("Discovering tests at %s" % testPath)
-                    testsDict[k] = testLoader.discover(testPath,
-                                                       pattern=args.pattern,
-                                                       top_level_dir=p)
-
-        self.grep = [g.lower() for g in args.grep] if args.grep else []
-        self.skip = [s.lower() for s in args.skip] if args.skip else []
-        self.mode = args.mode
-
-        if args.tests:
-            self.runSingleTest(testsDict['tests'])
-
-        elif args.run:
-            for moduleName, tests in testsDict.items():
-                self.logger.info(pwutils.cyanStr(">>>> %s" % moduleName))
-                self.runTests(moduleName, tests)
-
-        elif args.grep:
-            pattern = args.grep[0]
-            for moduleName, tests in testsDict.items():
-                self.printTests(pattern, tests)
-
-        else:
-            for moduleName, tests in testsDict.items():
-                if self._match(moduleName):
-                    print(pwutils.cyanStr(">>>> %s" % moduleName))
-                    self.printTests(moduleName, tests)
-
-    def _match(self, itemName):
-        # Add a space to allow " tomo." as a filter for example to narrow search to tomo and leaving out xmipptomo, emntomo, ...
-        itemLower = " " + itemName.lower()
-        grep = (not self.grep or
-                all(g in itemLower for g in self.grep))
-        skip = (self.skip and
-                any(s in itemLower for s in self.skip))
-
-        return grep and not skip
-
-    def __iterTests(self, test):
-        """ Recursively iterate over a testsuite. """
-        self.logger.debug("__iterTests: %s, is-suite: %s" % (str(test.__class__),
-                                                 isinstance(test, unittest.TestSuite)))
-
-        if isinstance(test, unittest.TestSuite):
-            for t in test:
-                self.__iterTests(t)
-        else:
-            yield test
-
-    def _visitTests(self, moduleName, tests, newItemCallback):
-        """ Show the list of tests available """
-        mode = self.mode
-
-        assert mode in ['modules', 'classes', 'onlyclasses', 'all'], 'Unknown mode %s' % mode
-
-        # First flatten the list of tests.
-        # testsFlat = list(iter(self.__iterTests(tests)))
-
-        testsFlat = []
-        toCheck = [t for t in tests]
-
-        while toCheck:
-            test = toCheck.pop()
-            if isinstance(test, unittest.TestSuite):
-                toCheck += [t for t in test]
-            elif test not in testsFlat:
-                testsFlat.append(test)
-
-        # Follow the flattened list of tests and show the module, class
-        # and name, in a nice way.
-        lastClass = None
-        lastModule = None
-        if testsFlat:
-            for t in testsFlat:
-
-                testModuleName, className, testName = t.id().rsplit('.', 2)
-
-                # If there is a failure loading the test, show it
-                errorStr = 'unittest.loader._FailedTest.'
-                if testModuleName.startswith(errorStr):
-                    newName = t.id().replace(errorStr, '')
-                    if self._match(newName):
-                        self.logger.error(
-                            pwutils.redStr('Error loading the test. Please, run the test for more information: ' + newName))
-                    continue
-
-                if testModuleName != lastModule:
-                    lastModule = testModuleName
-                    if mode != 'onlyclasses':
-                        newItemCallback(MODULE, "%s" % testModuleName)
-
-                if mode in ['classes', 'onlyclasses', 'all'] and className != lastClass:
-                    lastClass = className
-                    newItemCallback(CLASS, "%s.%s"
-                                    % (testModuleName, className))
-
-                if mode == 'all':
-                    newItemCallback(TEST, "%s.%s.%s"
-                                    % (testModuleName, className, testName))
-        else:
-            if not self.grep:
-                self.logger.warning(pwutils.greenStr(' The plugin does not have any test'))
-
-    def _printNewItem(self, itemType, itemName):
-        if self._match(itemName):
-            spaces = (itemType * 2) * ' '
-            # Do not use intentionally the logger. This is not a logging output but a GUI output
-            print("%s %s %s" % (spaces, self.getTestsCommand(), itemName))
-
-    def getTestsCommand(self):
-        return os.environ.get(SCIPION_TESTS_CMD, getTestsScript())
-
-    def printTests(self, moduleName, tests):
-        self._visitTests(moduleName, tests, self._printNewItem)
-
-    def _runNewItem(self, itemType, itemName):
-        if self._match(itemName):
-            spaces = (itemType * 2) * ' '
-            script = getTestsScript()
-            cmd = "%s %s %s %s" % (pw.PYTHON, script, spaces, itemName)
-            run = ((itemType == MODULE and self.mode == 'modules') or
-                   (itemType == CLASS and self.mode in ('classes', 'onlyclasses')) or
-                   (itemType == TEST and self.mode == 'all'))
-            if run:
-                if self.log:
-                    # logFile = join(self.testsDir, '%s.txt' % itemName)
-                    cmdFull = cmd + " >> %s 2>&1" % self.log
-                else:
-                    logFile = ''
-                    cmdFull = cmd
-
-                self.logger.info(pwutils.greenStr(cmdFull))
-                t = pwutils.Timer()
-                t.tic()
-                self.testCount += 1
-                os.system(cmdFull)
-
-    def runTests(self, moduleName, tests):
-
-        self.testCount = 0
-        self._visitTests(moduleName, tests, self._runNewItem)
-
-    def runSingleTest(self, tests):
-        result = pwtests.GTestResult()
-        tests.run(result)
-        result.doReport()
-        resultPassed = result.numberTests - result.testFailed
-        sys.exit(1 if result.testFailed > 0 or resultPassed == 0 else 0)
-
-
-if __name__ == '__main__':
-    Tester().main()

pyworkflow/apps/pw_schedule_run.py

@@ -1,322 +0,0 @@
-#!/usr/bin/env python
-# **************************************************************************
-# *
-# * Authors:     J.M. De la Rosa Trevin (delarosatrevin@scilifelab.se) [1]
-# *
-# * [1] SciLifeLab, Stockholm University
-# *
-# * This program is free software; you can redistribute it and/or modify
-# * it under the terms of the GNU General Public License as published by
-# * the Free Software Foundation; either version 3 of the License, or
-# * (at your option) any later version.
-# *
-# * This program is distributed in the hope that it will be useful,
-# * but WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# * GNU General Public License for more details.
-# *
-# * You should have received a copy of the GNU General Public License
-# * along with this program; if not, write to the Free Software
-# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# * 02111-1307  USA
-# *
-# *  All comments concerning this program package may be sent to the
-# *  e-mail address 'scipion@cnb.csic.es'
-# *
-# **************************************************************************
-
-import logging
-
-from pyworkflow.utils import LoggingConfigurator
-
-import os
-import sys
-import time
-import argparse
-
-from pyworkflow.protocol import (getProtocolFromDb,
-                                 STATUS_FINISHED, STATUS_ABORTED, STATUS_FAILED,
-                                 STATUS_RUNNING, STATUS_SCHEDULED, STATUS_SAVED,
-                                 STATUS_LAUNCHED, Set, Protocol, MAX_SLEEP_TIME)
-from pyworkflow.constants import PROTOCOL_UPDATED
-
-stopStatuses = [STATUS_FINISHED, STATUS_ABORTED, STATUS_FAILED]
-
-
-class RunScheduler:
-    """ Check that all dependencies are met before launching a run. """
-
-    def __init__(self, args, logger):
-        self._args = args
-        # Enter to the project directory and load protocol from db
-        self.protocol = self._loadProtocol()
-        self.project = self.protocol.getProject()
-        self.log = logger
-        self.protPid = os.getpid()
-        self.protocol.setPid(self.protPid)
-        self.protocol._store(self.protocol._pid)
-        self.prerequisites = list(map(int, self.protocol.getPrerequisites()))
-        # Keep track of the last time the protocol was checked and
-        # its modification date to avoid unnecessary db opening
-        self.updatedProtocols = dict()
-        self.initial_sleep = self._args.initial_sleep
-
-    def getSleepTime(self):
-        return self._args.sleepTime
-
-    def getInitialSleepTime(self):
-        return self.initial_sleep
-
-    def _loadProtocol(self) -> Protocol:
-        return getProtocolFromDb(self._args.projPath,
-                                 self._args.dbPath,
-                                 self._args.protId, chdir=True)
-
-    def _log(self, msg):
-        self.log.info(msg)
-
-    def _updateProtocol(self, protocol):
-
-        protId = protocol.getObjId()
-
-        if protId in self.updatedProtocols:
-            return self.updatedProtocols[protId]
-
-        protDb = protocol.getDbPath()
-
-        if os.path.exists(protDb):
-            updateResult = self.project._updateProtocol(protocol)
-            if updateResult == PROTOCOL_UPDATED:
-                self._log("Updated protocol: %s (%s)" % (protId, protocol))
-            else:
-                self._log("The protocol %s (%s) is up to date" % (protId, protocol))
-            self.updatedProtocols[protId] = protocol
-
-        return protocol
-
-    def _getProtocolFromPointer(self, pointer):
-        """
-        The function return a protocol from an attribute
-
-           A) When the pointer points to a protocol
-
-           B) When the pointer points to another object (INDIRECTLY).
-              - The pointer has an _extended value (new parameters
-                configuration in the protocol)
-
-           C) When the pointer points to another object (DIRECTLY).
-              - The pointer has not an _extended value (old parameters
-                configuration in the protocol)
-        """
-        output = pointer.get()
-        if isinstance(output, Protocol):  # case A
-            protocol = output
-        else:
-            if pointer.hasExtended():  # case B
-                protocol = pointer.getObjValue()
-            else:  # case C
-                protocol = self.project.getRunsGraph().getNode(str(output.getObjParentId())).run
-        return protocol
-
-    def _getSecondsToWait(self, inProt):
-        """
-        Assigns a timeout penalty or reward depending on the status of the
-        input protocol (inProt)
-        If inProt status is stopped we assign a reward i.o.c a penalty
-        """
-        protStatus = inProt.getStatus()
-        inStreaming = inProt.worksInStreaming()
-        meStreaming = self.protocol.worksInStreaming()
-
-        penaltyRewardValues = {
-            STATUS_LAUNCHED: 5 if inStreaming else 10,
-            STATUS_RUNNING: -2 if inStreaming else 3,
-            STATUS_SCHEDULED: int(self.getSleepTime()/2),
-            STATUS_SAVED: self.getSleepTime(),
-        }
-
-        secondToWait = penaltyRewardValues.get(protStatus, -3)
-
-        if not meStreaming:
-            secondToWait += 3 * self.getSleepTime()
-
-        return secondToWait
-
-    def _checkPrerequisites(self, prerequisites, project):
-        # Check if we need to wait for required protocols
-        wait = False
-        # For each protocol that is a prerequisite that has not finished,
-        # we'll penalize with 3 more seconds of waiting
-        penalize = 0
-        self._log("Checking prerequisites... %s" % prerequisites)
-        for protId in prerequisites:
-            # Check if prerequisites exist. In the case of metaprotocols, it
-            # may be necessary to load them from the project database.
-            node = project.getRunsGraph().getNode(str(protId))
-
-            if node is None:
-                self._log("Updating runs graph. Missing protocol ... %s" % protId)
-                project.getRunsGraph(refresh=True)
-
-            node = project.getRunsGraph().getNode(str(protId))
-            # Check if the protocol is within our workflow
-            if node is None:
-                self._log("Protocol can't wait for %s. Missing prerequisite " % protId)
-                break
-
-            prot = project.getRunsGraph().getNode(str(protId)).run
-            if prot is not None:
-                prot = self._updateProtocol(prot)
-                penalize += self._getSecondsToWait(prot)
-                if prot.getStatus() not in stopStatuses:
-                    wait = True
-                    self._log("   ...waiting for %s" % prot)
-        return wait, penalize
-
-    def _checkMissingInput(self):
-        """
-         Check if there are missing inputs
-         In case of having them, check if protocol is in streaming
-        """
-        inputMissing = False
-        # For each input that is not ready we'll penalize with 3 more
-        # seconds of waiting
-        penalize = 0
-
-        self._log("Checking input data...")
-        # Updating input protocols
-        for key, attr in self.protocol.iterInputAttributes():
-            self.log.debug("Turn for %s" % key)
-            inputProt = self._getProtocolFromPointer(attr)
-            inputProt = self._updateProtocol(inputProt)
-            penalize += self._getSecondsToWait(inputProt)
-
-        validation = self.protocol.validate()
-        if len(validation) > 0:
-            inputMissing = True
-            self._log("%s doesn't validate:\n\t- %s" % (self.protocol.getObjLabel(),
-                                                        '\n\t- '.join(
-                                                            validation)))
-        elif not self.protocol.worksInStreaming():
-            for key, attr in self.protocol.iterInputAttributes():
-                inSet = attr.get()
-                if isinstance(inSet, Set) and inSet.isStreamOpen():
-                    inputMissing = True
-                    self._log("Waiting for closing %s... (does not work in "
-                              "streaming)" % inSet)
-                    break
-                elif isinstance(inSet, Protocol) and not inSet.isFinished():  # Then is a pointer to a protocol
-                    inputMissing = True
-                    self._log("Waiting for protocol %s to finish... (does not work in "
-                              "streaming)" % inSet)
-                    break
-
-                else:
-                    self._log("%s is not blocking this protocol. All ready." % inSet)
-
-        if not inputMissing:
-            inputProtocolIds = self.protocol.getProtocolsToUpdate()
-            for protId in inputProtocolIds:
-                protocol = self.project.getProtocol(protId)
-                self.log.debug("Turn from inputProtocolDict for %s" % protocol)
-                self._updateProtocol(protocol)
-
-        return inputMissing, penalize
-
-    def schedule(self):
-        self._log("Scheduling protocol %s, PID: %s,prerequisites: %s, works in streaming: %s." %
-                  (self.protocol.getObjId(), self.protPid, self.prerequisites,
-                   self.protocol.worksInStreaming()))
-
-        initialSleepTime = runScheduler.getInitialSleepTime()
-        self._log("Waiting %s seconds before start checking inputs " % initialSleepTime)
-        time.sleep(initialSleepTime)
-
-        while True:
-            # Clear the list of protocols updated in the previous loop
-            self.updatedProtocols.clear()
-            sleepTime = self.getSleepTime()
-            # FIXME: This does not cover all the cases:
-            # When the user registers new coordinates after clicking the
-            # "Analyze result" button, this action is registered in the project.sqlite
-            # and not in it's own run.db and never gets updated. It is not critical and
-            # will only affect a combination of json import with extended that will
-            # appear after clicking on the "Analyze result" button.
-
-            # Check if there are missing inputs
-            missing, penalize = self._checkMissingInput()
-            sleepTime += penalize
-
-            # Check the prerequisites
-            wait, penalize = self._checkPrerequisites(self.prerequisites,
-                                                      self.project)
-            sleepTime += penalize
-
-            if not missing and not wait:
-                break
-
-            sleepTime = max(min(sleepTime, MAX_SLEEP_TIME), self.getSleepTime())
-
-            self._log("Still not ready, sleeping %s seconds...\n" % sleepTime)
-            time.sleep(sleepTime)
-
-        self._log("Launching the protocol >>>>")
-        self.project.launchProtocol(self.protocol, scheduled=True, force=True)
-
-def parseArgs():
-    parser = argparse.ArgumentParser()
-    _addArg = parser.add_argument  # short notation
-
-    _addArg("projPath", metavar='PROJECT_NAME',
-            help="Project database path.")
-
-    _addArg("dbPath", metavar='DATABASE_PATH',
-            help="Protocol database path.")
-
-    _addArg("protId", type=int, metavar='PROTOCOL_ID',
-            help="Protocol ID.")
-
-    _addArg("logPath", metavar='LOG_PATH', help='Path to the log file.')
-
-    _addArg("--initial_sleep", type=int, default=0,
-            dest='initial_sleep', metavar='SECONDS',
-            help="Initial sleeping time (in seconds)")
-
-    _addArg("--sleep_time", type=int, default=15,
-            dest='sleepTime', metavar='SECONDS',
-            help="Sleeping time (in seconds) between updates.")
-
-    _addArg("--wait_for", nargs='*', type=int, default=[],
-            dest='waitProtIds', metavar='PROTOCOL_ID',
-            help="List of protocol ids that should be not running "
-                 "(i.e, finished, aborted or failed) before this "
-                 "run will be executed.")
-
-    return parser.parse_args()
-
-
-if __name__ == '__main__':
-
-    # Create a child process
-    # using os.fork() method
-    pid = os.fork()
-
-    # pid greater than 0 represents
-    # the parent process
-    if pid > 0:
-        sys.exit(0)
-    else:
-        try:
-            # Parse arguments
-            args = parseArgs()
-
-            # Configure logging
-            LoggingConfigurator.setUpProtocolSchedulingLog(args.logPath)
-            logger = logging.getLogger(__name__)
-
-            runScheduler = RunScheduler(args, logger)
-            runScheduler.schedule()
-
-        except Exception as ex:
-            print("Scheduling failed with these parameters: ", sys.argv)
-            raise ex from None

pyworkflow/apps/pw_sleep.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# **************************************************************************
-# *
-# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
-# *
-# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
-# *
-# * This program is free software; you can redistribute it and/or modify
-# * it under the terms of the GNU General Public License as published by
-# * the Free Software Foundation; either version 3 of the License, or
-# * (at your option) any later version.
-# *
-# * This program is distributed in the hope that it will be useful,
-# * but WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# * GNU General Public License for more details.
-# *
-# * You should have received a copy of the GNU General Public License
-# * along with this program; if not, write to the Free Software
-# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
-# * 02111-1307  USA
-# *
-# *  All comments concerning this program package may be sent to the
-# *  e-mail address 'scipion@cnb.csic.es'
-# *
-# **************************************************************************
-"""
-This module is responsible for launching protocol executions.
-"""
-import os
-import sys
-
-if __name__ == '__main__':
-    seconds = sys.argv[1]
-    print("Sleeper, pid: ", os.getpid())
-    print("Sleeping %s seconds...." % seconds)
-    os.system("sleep %s" % seconds)