scipion-pyworkflow 3.11.0__py3-none-any.whl → 3.11.1__py3-none-any.whl

pyworkflow/project/usage.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# **************************************************************************
+# *
+# * Authors:   Pablo Conesa [1]
+# *
+# * [1] Biocomputing unit, CNB-CSIC
+# *
+# * This program is free software: you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation, either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+import json
+import logging
+logger = logging.getLogger(__name__)
+
+from urllib.request import urlopen
+from pyworkflow import Config
+
+
+# Module to have Models for reporting usage data to the scipion.i2pc.es site.
+class ProtStat:
+    """ Class to store the usage part of a reported ScipionWorkflow"""
+    def __init__(self, count=0, nextProtsDict=None):
+        self._count = count
+        self._nextProts = nextProtsDict or dict()
+
+    def getCount(self):
+        return self._count
+    def addUsage(self, count=1):
+        self._count += count
+    def addCountToNextProtocol(self, nextProt, count=1):
+        self._nextProts[nextProt] = self._nextProts.get(nextProt, 0) + count
+
+    def toJSON(self):
+        jsonStr = "[%s,{%s}]"
+        nextProtS = ""
+
+        if len(self._nextProts):
+            nextProtA = []
+            for protName, count in self._nextProts.items():
+                nextProtA.append('"%s":%d' % (protName, count))
+
+            nextProtS= ",".join(nextProtA)
+
+        jsonStr = jsonStr % (self._count, nextProtS)
+
+        return jsonStr
+
+    def __repr__(self):
+        return self.toJSON()
+
+class ScipionWorkflow:
+    """ Class to serialize and deserialize what is reported from scipion.
+    Example: {"ProtA":
+                [2, {
+                        "ProtB":2,
+                        "ProtC":3,
+                        ...
+                    }
+                ], ...
+              }
+    """
+
+    def __init__(self, jsonStr=None):
+        """ Instantiate this class optionally with a JSON string serialized from this class
+        (what is sent by Scipion to this web service)."""
+
+        self._prots = dict()
+        if jsonStr is not None:
+            self.deserialize(jsonStr)
+
+    def getProtStats(self):
+        return self._prots
+
+    def deserialize(self, jsonStr):
+        """ Deserialize a JSONString serialized by this class with the toJSON method"""
+        jsonObj = json.loads(jsonStr)
+
+        if isinstance(jsonObj, dict):
+            self.deserializeV2(jsonObj)
+        else:
+            self.deserializeList(jsonObj)
+
+    def deserializeV2(self, jsonObj):
+        """ Deserializes v2 usage stats: something like {"ProtA": [2,{..}],...} """
+        for key, value in jsonObj.items():
+            # Value should be something like [2,{..}]
+            count = value[0]
+            nextProtDict = value[1]
+            nextProt = ProtStat(count, nextProtDict)
+            self._prots[key] = nextProt
+
+    def deserializeList(self, jsonObj):
+        """ Deserializes old data: a list of protocol names repeated: ["ProtA","ProtA", "ProtB", ...] """
+
+        for protName in jsonObj:
+            self.addCount(protName)
+
+    def addCount(self, protName):
+        """ Adds one to the count of a protocol"""
+
+        protStat = self.getProtStat(protName)
+
+        protStat.addUsage()
+
+    def getProtStat(self, protName):
+
+        protStat = self._prots.get(protName, ProtStat())
+        if protName not in self._prots:
+            self._prots[protName] = protStat
+
+        return protStat
+
+    def addCountToNextProtocol(self, protName, nextProtName):
+        protStat = self.getProtStat(protName)
+        protStat.addCountToNextProtocol(nextProtName)
+
+    def getCount(self):
+        """ Returns the number of protocols in the workflow"""
+        count = 0
+        for ps in self._prots.values():
+            count += ps._count
+        return count
+
+    def toJSON(self):
+        """ Returns a valid JSON string"""
+        if len(self._prots) == 0:
+            return "{}"
+        else:
+            jsonStr="{"
+            for protName, protStat in self._prots.items():
+
+                jsonStr += '"%s":%s,' % (protName, protStat.toJSON())
+
+            jsonStr = jsonStr[:-1] + "}"
+
+            return jsonStr
+
+    def __repr__(self):
+        return self.toJSON()
+
+def getNextProtocolSuggestions(protocol):
+    """ Returns the suggestions from the Scipion website for the next protocols to the protocol passed"""
+
+    try:
+        url = Config.SCIPION_STATS_SUGGESTION % protocol  # protocol.getClassName()
+        results = json.loads(urlopen(url).read().decode('utf-8'))
+        return results
+    except Exception as e:
+        logger.error("Suggestions system not available", exc_info=e)
+        return []

pyworkflow/protocol/__init__.py

@@ -0,0 +1,38 @@
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
+# *
+# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+"""
+This modules contains classes required for the workflow
+execution and tracking like: Step and Protocol
+"""
+
+from .protocol import *
+from .executor import *
+from .constants import *
+from .params import *
+
+from .launch import *
+
+from .hosts import HostConfig

pyworkflow/protocol/bibtex.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# **************************************************************************
+# *
+# * Authors:     Yaiza Rancel (cyrancel@cnb.csic.es)
+# *
+# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+"""
+@article{delaRosaTrevin201693,
+title = "Scipion: A software framework toward integration, reproducibility and validation in 3D electron microscopy ",
+journal = "Journal of Structural Biology",
+volume = "195",
+number = "1",
+pages = "93 - 99",
+year = "2016",
+note = "",
+issn = "1047-8477",
+doi = "http://doi.org/10.1016/j.jsb.2016.04.010",
+url = "http://www.sciencedirect.com/science/article/pii/S104784771630079X",
+author = "J.M. de la Rosa-Trevín and A. Quintana and L. del Cano and A. Zaldívar and I. Foche and J. Gutiérrez and J. Gómez-Blanco and J. Burguet-Castell and J. Cuenca-Alba and V. Abrishami and J. Vargas and J. Otón and G. Sharov and J.L. Vilas and J. Navas and P. Conesa and M. Kazemi and R. Marabini and C.O.S. Sorzano and J.M. Carazo",
+keywords = "Electron microscopy",
+keywords = "Single particle analysis",
+keywords = "Image processing",
+keywords = "Software package",
+keywords = "Workflows",
+keywords = "Reproducibility "
+}
+"""
+

pyworkflow/protocol/constants.py

@@ -0,0 +1,87 @@
+# **************************************************************************
+# *
+# * Authors:     J.M. De la Rosa Trevin (jmdelarosa@cnb.csic.es)
+# *
+# * Unidad de  Bioinformatica of Centro Nacional de Biotecnologia , CSIC
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 3 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+"""
+This modules contains classes required for the workflow
+execution and tracking like: Step and Protocol
+"""
+
+# ------------------ Constants values -----------------------------------------
+
+# Possible status of a protocol run, used mainly to monitor progress
+
+STATUS_SAVED = "saved"  # Parameters saved for later use
+STATUS_LAUNCHED = "launched"  # launched to queue system, only useful for protocols
+STATUS_NEW = "new"
+STATUS_RUNNING = "running"    # currently executing
+STATUS_FAILED = "failed"      # it run and failed
+STATUS_FINISHED = "finished"  # successfully finished
+STATUS_ABORTED = "aborted"
+STATUS_INTERACTIVE = "interactive"    # waiting for user interaction
+STATUS_WAITING = "waiting"            # following status is used for streaming only
+# Allow to schedule jobs not ready for running yet
+STATUS_SCHEDULED = "scheduled"
+
+ACTIVE_STATUS = [STATUS_LAUNCHED, STATUS_RUNNING, STATUS_INTERACTIVE,
+                 STATUS_SCHEDULED]
+
+# Execution modes
+MODE_RESUME = 0    # Try to starting at the first changed step, skipping unchanged ones
+MODE_RESTART = 1   # Restart the protocol from the beginning, deleting all previous results
+MODE_CONTINUE = 2  # Continue from specific step, not widely used //DEPRECATED.
+# JMRT: We now use 'Continue' label instead of 'Resume' which is more intuitive for users.
+MODE_CHOICES = ('Continue', 'Restart')  # , 'Continue')
+
+# Initial sleeping time (in seconds) in order to launch a scheduled protocol.
+# This value is multiply to the protocol level into the workflow.
+INITIAL_SLEEP_TIME = 30
+
+# Maximum time (in seconds) waiting for a scheduled protocol to check if it
+# can be launched.
+MAX_SLEEP_TIME = 120
+
+
+# Steps execution mode
+STEPS_SERIAL = 0      # Execute steps serially, some of the steps can be mpi programs
+STEPS_PARALLEL = 1    # Execute steps in parallel through threads
+
+# Level of expertise for the input parameters, mainly used in the protocol form
+LEVEL_NORMAL = 0
+LEVEL_ADVANCED = 1
+LEVEL_CHOICES = ('Normal', 'Advanced')
+
+# Param names for GPU processing
+USE_GPU = 'useGpu'
+GPU_LIST = 'gpuList'
+VOID_GPU = 99
+
+# Job management
+UNKNOWN_JOBID = -1
+
+# File sizes
+SIZE_1KB = 1024
+SIZE_1MB = SIZE_1KB * SIZE_1KB
+SIZE_1GB = SIZE_1MB * SIZE_1KB
+SIZE_1TB = SIZE_1GB * SIZE_1KB